From db786e6c6f144c794ee923328fef02c6c6618d2e Mon Sep 17 00:00:00 2001 From: Daniel Thomas Date: Wed, 5 Feb 2025 16:46:14 +0100 Subject: [PATCH] feat: initial boostrapping of api structure --- .editorconfig | 12 +++++ .golangci.yml | 60 +++++++++++++++++++++ .goreleaser.yml | 34 ++++++++++++ Makefile | 91 ++++++++++++++++++++++++++++++++ README.md | 0 go.mod | 16 ++++++ go.sum | 35 +++++++++++++ pkg/api/api.go | 121 +++++++++++++++++++++++++++++++++++++++++++ pkg/api/interface.go | 24 +++++++++ pkg/api/type.go | 13 +++++ pkg/api/utils.go | 42 +++++++++++++++ 11 files changed, 448 insertions(+) create mode 100644 .editorconfig create mode 100644 .golangci.yml create mode 100644 .goreleaser.yml create mode 100644 Makefile create mode 100644 README.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 pkg/api/api.go create mode 100644 pkg/api/interface.go create mode 100644 pkg/api/type.go create mode 100644 pkg/api/utils.go diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..b5dd3b6 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_size = 2 +indent_style = tab +insert_final_newline = true +trim_trailing_whitespace = true + +[*.{yaml,yml,md,mdx}] +indent_style = space \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..142de22 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,60 @@ +run: + timeout: 5m + +linters-settings: + importas: + alias: + - pkg: '^github.com\/foomo\/typesense\/(.*\/)?([^\/]+)\/?$' + alias: '${2}x' # enforce `x` suffix + no-unaliased: true + +linters: + enable: + # Enabled by default linters: + - errcheck + - gosimple + - govet + - ineffassign + - staticcheck + + # Disabled by default linters: + - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers [fast: true, auto-fix: false] + - bidichk # Checks for dangerous unicode character sequences [fast: true, auto-fix: false] + # - depguard # Go linter that checks if package imports are in a list of acceptable packages [fast: true, auto-fix: false] + # - dupl # Tool for code clone detection [fast: true, auto-fix: false] + - forcetypeassert # finds forced type assertions [fast: true, auto-fix: false] + - gochecknoinits # Checks that no init functions are present in Go code [fast: true, auto-fix: false] + - goconst # Finds repeated strings that could be replaced by a constant [fast: true, auto-fix: false] + - gocritic # Provides diagnostics that check for bugs, performance and style issues. [fast: false, auto-fix: false] + - goimports # In addition to fixing imports, goimports also formats your code in the same style as gofmt. [fast: true, auto-fix: true] + - gomoddirectives # Manage the use of 'replace', 'retract', and 'excludes' directives in go.mod. [fast: true, auto-fix: false] + - gomodguard # Allow and block list linter for direct Go module dependencies. This is different from depguard where there are different block types for example version constraints and module recommendations. [fast: true, auto-fix: false] + - goprintffuncname # Checks that printf-like functions are named with `f` at the end [fast: true, auto-fix: false] + - gosec # (gas): Inspects source code for security problems [fast: false, auto-fix: false] + - grouper # An analyzer to analyze expression groups. [fast: true, auto-fix: false] + # - importas # Enforces consistent import aliases [fast: false, auto-fix: false] + - maintidx # maintidx measures the maintainability index of each function. [fast: true, auto-fix: false] + - makezero # Finds slice declarations with non-zero initial length [fast: false, auto-fix: false] + - misspell # Finds commonly misspelled English words in comments [fast: true, auto-fix: true] + - nakedret # Finds naked returns in functions greater than a specified function length [fast: true, auto-fix: false] + - nestif # Reports deeply nested if statements [fast: true, auto-fix: false] + - nilerr # Finds the code that returns nil even if it checks that the error is not nil. [fast: false, auto-fix: false] + - nilnil # Checks that there is no simultaneous return of `nil` error and an invalid value. [fast: false, auto-fix: false] + - noctx # noctx finds sending http request without context.Context [fast: false, auto-fix: false] + - nolintlint # Reports ill-formed or insufficient nolint directives [fast: true, auto-fix: false] + # - nonamedreturns # Reports all named returns [fast: false, auto-fix: false] + - nosprintfhostport # Checks for misuse of Sprintf to construct a host with port in a URL. [fast: true, auto-fix: false] + - prealloc # Finds slice declarations that could potentially be pre-allocated [fast: true, auto-fix: false] + - predeclared # find code that shadows one of Go's predeclared identifiers [fast: true, auto-fix: false] + - promlinter # Check Prometheus metrics naming via promlint [fast: true, auto-fix: false] + - revive # Fast, configurable, extensible, flexible, and beautiful linter for Go. Drop-in replacement of golint. [fast: false, auto-fix: false] + - tagliatelle # Checks the struct tags. [fast: true, auto-fix: false] + - testpackage # linter that makes you use a separate _test package [fast: true, auto-fix: false] + - thelper # thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers [fast: false, auto-fix: false] + - unconvert # Remove unnecessary type conversions [fast: false, auto-fix: false] + - unparam # Reports unused function parameters [fast: false, auto-fix: false] + - usestdlibvars # A linter that detect the possibility to use variables/constants from the Go standard library. [fast: true, auto-fix: false] + - wastedassign # wastedassign finds wasted assignment statements. [fast: false, auto-fix: false] + - whitespace # Tool for detection of leading and trailing whitespace [fast: true, auto-fix: true] + disable: + - unused diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..65027d3 --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,34 @@ +builds: + - skip: true + +changelog: + filters: + exclude: + - "^wip" + - "^test" + - "^docs" + - "^chore" + - "^style" + - "go mod tidy" + - "merge conflict" + - "Merge pull request" + - "Merge remote-tracking branch" + - "Merge branch" + groups: + - title: Features + regexp: '^.*?feat(\([[:word:]]+\))??!?:.+$' + order: 0 + - title: Dependency updates + regexp: '^.*?(feat|fix)\(deps\)!?:.+$' + order: 100 + - title: "Bug fixes" + regexp: '^.*?fix(\([[:word:]]+\))??!?:.+$' + order: 150 + - title: "Security" + regexp: '^.*?sec(\([[:word:]]+\))??!?:.+$' + order: 200 + - title: "Performace" + regexp: '^.*?perf(\([[:word:]]+\))??!?:.+$' + order: 250 + - title: Other + order: 999 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..26b0f06 --- /dev/null +++ b/Makefile @@ -0,0 +1,91 @@ +.DEFAULT_GOAL:=help +-include .makerc + +# --- Targets ----------------------------------------------------------------- + +# This allows us to accept extra arguments +%: .husky + @: + +.PHONY: .husky +# Configure git hooks for husky +.husky: + @if ! command -v husky &> /dev/null; then \ + echo "ERROR: missing executeable 'husky', please run:"; \ + echo "\n$ go install github.com/go-courier/husky/cmd/husky@latest\n"; \ + fi + @git config core.hooksPath .husky + +## === Tasks === + +.PHONY: doc +## Run tests +doc: + @open "http://localhost:6060/pkg/github.com/foomo/typesense/" + @godoc -http=localhost:6060 -play + +.PHONY: test +## Run tests +test: + @set -euo pipefail && go test -json -v ./... | gotestfmt + +.PHONY: test.cover +## Run tests with coverage +test.cover: + @go test -v -coverprofile=coverage.out ./... + @go tool cover -func=coverage.out + @go tool cover -html=coverage.out + +.PHONY: lint +## Run linter +lint: + @golangci-lint run + +.PHONY: lint.fix +## Run linter and fix violations +lint.fix: + @golangci-lint run --fix + +.PHONY: tidy +## Run go mod tidy +tidy: + @go mod tidy + +.PHONY: outdated +## Show outdated direct dependencies +outdated: + @go list -u -m -json all | go-mod-outdated -update -direct + +## === Utils === + +## Show help text +help: + @awk '{ \ + if ($$0 ~ /^.PHONY: [a-zA-Z\-\_0-9]+$$/) { \ + helpCommand = substr($$0, index($$0, ":") + 2); \ + if (helpMessage) { \ + printf "\033[36m%-23s\033[0m %s\n", \ + helpCommand, helpMessage; \ + helpMessage = ""; \ + } \ + } else if ($$0 ~ /^[a-zA-Z\-\_0-9.]+:/) { \ + helpCommand = substr($$0, 0, index($$0, ":")); \ + if (helpMessage) { \ + printf "\033[36m%-23s\033[0m %s\n", \ + helpCommand, helpMessage"\n"; \ + helpMessage = ""; \ + } \ + } else if ($$0 ~ /^##/) { \ + if (helpMessage) { \ + helpMessage = helpMessage"\n "substr($$0, 3); \ + } else { \ + helpMessage = substr($$0, 3); \ + } \ + } else { \ + if (helpMessage) { \ + print "\n "helpMessage"\n" \ + } \ + helpMessage = ""; \ + } \ + }' \ + $(MAKEFILE_LIST) diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..8423c46 --- /dev/null +++ b/go.mod @@ -0,0 +1,16 @@ +module github.com/foomo/typesense + +go 1.23.4 + +require ( + github.com/typesense/typesense-go/v3 v3.0.0 + go.uber.org/zap v1.27.0 +) + +require ( + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/oapi-codegen/runtime v1.1.1 // indirect + github.com/sony/gobreaker v1.0.0 // indirect + go.uber.org/multierr v1.10.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..59d0885 --- /dev/null +++ b/go.sum @@ -0,0 +1,35 @@ +github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= +github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jinzhu/copier v0.3.4 h1:mfU6jI9PtCeUjkjQ322dlff9ELjGDu975C2p/nrubVI= +github.com/jinzhu/copier v0.3.4/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= +github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= +github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sony/gobreaker v1.0.0 h1:feX5fGGXSl3dYd4aHZItw+FpHLvvoaqkawKjVNiFMNQ= +github.com/sony/gobreaker v1.0.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/typesense/typesense-go/v3 v3.0.0 h1:uLCMfVhv5GkZNjMGr/UHqVMKKdF0bkoTH4hAeigw8PE= +github.com/typesense/typesense-go/v3 v3.0.0/go.mod h1:Jx4PAXe3jRx6sc032nhN9Aj+OvMoPtQJW6p1a6H4Zeg= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= +go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/api/api.go b/pkg/api/api.go new file mode 100644 index 0000000..7d6a434 --- /dev/null +++ b/pkg/api/api.go @@ -0,0 +1,121 @@ +package typesenseapi + +import ( + "context" + "errors" + + "github.com/typesense/typesense-go/v3/typesense" + "go.uber.org/zap" + + "github.com/typesense/typesense-go/v3/typesense/api" +) + +const defaultSearchPresetName = "default" + +type BaseAPI[indexDocument any, returnDocument any] struct { + l *zap.Logger + client *typesense.Client + collections map[IndexID]*api.CollectionSchema + preset *api.PresetUpsertSchema + + revisionID RevisionID +} + +func NewBaseAPI[indexDocument any, returnDocument any]( + l *zap.Logger, + client *typesense.Client, + collections map[IndexID]*api.CollectionSchema, + preset *api.PresetUpsertSchema, +) *BaseAPI[indexDocument, returnDocument] { + return &BaseAPI[indexDocument, returnDocument]{ + l: l, + client: client, + collections: collections, + preset: preset, + } +} + +// Healthz will check if the revisionID is set +func (b *BaseAPI[indexDocument, returnDocument]) Healthz(_ context.Context) error { + if b.revisionID == "" { + return errors.New("revisionID not set") + } + return nil +} + +// Initialize +// will check the typesense connection and state of the colllections and aliases +// if the collections and aliases are not in the correct state it will create new collections and aliases +// +// example: +// +// b.collections := map[IndexID]*api.CollectionSchema{ +// "www-bks-at-de": { +// Name: "www-bks-at-de", +// }, +// "digital-bks-at-de": { +// Name: "digital-bks-at-de", +// } +// } +// +// there should be 2 aliases "www-bks-at-de" and "digital-bks-at-de" +// there should be at least 2 collections one for each alias +// the collection names are concatenated with the revisionID: "www-bks-at-de-2021-01-01-12" +// the revisionID is a timestamp in the format "YYYY-MM-DD-HH". If multiple collections are available +// the latest revisionID can be identified by the latest timestamp value +// +// Additionally, make sure that the configured search preset is present +// The system is ok if there is one alias for each collection and the collections are linked to the correct alias +// The function will set the revisionID that is currently linked to the aliases internally +func (b *BaseAPI[indexDocument, returnDocument]) Initialize() error { + var revisionID RevisionID + // use b.client.Health() to check the connection + + b.revisionID = revisionID + return nil +} + +func (b *BaseAPI[indexDocument, returnDocument]) NewRevision() (RevisionID, error) { + var revision RevisionID + + // create a revisionID based on the current time "YYYY-MM-DD-HH" + + // for all b.collections + // create a new collection in typesense - IndexID + - + revisionID + return revision, nil +} + +func (b *BaseAPI[indexDocument, returnDocument]) UpsertDocuments( + revisionID RevisionID, + indexID IndexID, + documents []indexDocument, +) error { + // use api to upsert documents + return nil +} + +// CommitRevision this is called when all the documents have been upserted +// it will update the aliases to point to the new revision +// additionally it will remove all old collections that are not linked to an alias +// keeping only the latest revision and the one before +func (b *BaseAPI[indexDocument, returnDocument]) CommitRevision(revisionID RevisionID) error { + return nil +} + +// SimpleSearch will perform a search operation on the given index +// it will return the documents and the scores +func (b *BaseAPI[indexDocument, returnDocument]) SimpleSearch( + index IndexID, + q string, + filterBy map[string]string, + page, perPage int, + sortBy string, +) ([]returnDocument, Scores, error) { + return b.ExpertSearch(index, getSearchCollectionParameters(q, filterBy, page, perPage, sortBy)) +} + +// ExpertSearch will perform a search operation on the given index +// it will return the documents and the scores +func (b *BaseAPI[indexDocument, returnDocument]) ExpertSearch(index IndexID, parameters *api.SearchCollectionParams) ([]returnDocument, Scores, error) { + return nil, nil, nil +} diff --git a/pkg/api/interface.go b/pkg/api/interface.go new file mode 100644 index 0000000..997a357 --- /dev/null +++ b/pkg/api/interface.go @@ -0,0 +1,24 @@ +package typesenseapi + +import "github.com/typesense/typesense-go/v3/typesense/api" + +type API[indexDocument any, returnDocument any] interface { + // this will prepare new indices with the given schema and the index IDs configured for the API + NewRevision() (RevisionID, error) + CommitRevision(revisionID RevisionID) error + UpsertDocuments(revisionID RevisionID, indexID IndexID, documents []indexDocument) error + + // this will check the typesense connection and initialize the indices + // should be run directly in a main.go or similar to ensure the connection is working + Initialize() (RevisionID, error) + + // perform a search operation on the given index + SimpleSearch( + index IndexID, + q string, + filterBy map[string]string, + page, perPage int, + sortBy string, + ) ([]returnDocument, Scores, error) + ExpertSearch(index IndexID, parameters *api.SearchCollectionParams) ([]returnDocument, Scores, error) +} diff --git a/pkg/api/type.go b/pkg/api/type.go new file mode 100644 index 0000000..be8be44 --- /dev/null +++ b/pkg/api/type.go @@ -0,0 +1,13 @@ +package typesenseapi + +type RevisionID string +type Query string +type IndexID string +type DocumentID string + +type Scores map[DocumentID]Score + +type Score struct { + ID DocumentID + Index int +} diff --git a/pkg/api/utils.go b/pkg/api/utils.go new file mode 100644 index 0000000..b985efa --- /dev/null +++ b/pkg/api/utils.go @@ -0,0 +1,42 @@ +package typesenseapi + +import ( + "strings" + + "github.com/typesense/typesense-go/v3/typesense/api" + "github.com/typesense/typesense-go/v3/typesense/api/pointer" +) + +// getSearchCollectionParameters will return the search collection parameters +// this is meant as a utility function to create the search collection parameters +// for the typesense search API without any knowledge of the typesense API +func getSearchCollectionParameters( + q string, + filterBy map[string]string, + page, perPage int, + sortBy string, +) *api.SearchCollectionParams { + parameters := &api.SearchCollectionParams{} + parameters.Q = pointer.String(q) + if filterByString := getFilterByString(filterBy); filterByString != "" { + parameters.FilterBy = pointer.String(filterByString) + } + parameters.Page = pointer.Int(page) + parameters.PerPage = pointer.Int(perPage) + if sortBy != "" { + parameters.SortBy = pointer.String(sortBy) + } + + return parameters +} + +func getFilterByString(filterBy map[string]string) string { + if filterBy == nil { + return "" + } + filterByString := []string{} + for key, value := range filterBy { + filterByString = append(filterByString, key+":="+value) + } + return strings.Join(filterByString, "&&") +}