diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..b5dd3b6 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_size = 2 +indent_style = tab +insert_final_newline = true +trim_trailing_whitespace = true + +[*.{yaml,yml,md,mdx}] +indent_style = space \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..142de22 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,60 @@ +run: + timeout: 5m + +linters-settings: + importas: + alias: + - pkg: '^github.com\/foomo\/typesense\/(.*\/)?([^\/]+)\/?$' + alias: '${2}x' # enforce `x` suffix + no-unaliased: true + +linters: + enable: + # Enabled by default linters: + - errcheck + - gosimple + - govet + - ineffassign + - staticcheck + + # Disabled by default linters: + - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers [fast: true, auto-fix: false] + - bidichk # Checks for dangerous unicode character sequences [fast: true, auto-fix: false] + # - depguard # Go linter that checks if package imports are in a list of acceptable packages [fast: true, auto-fix: false] + # - dupl # Tool for code clone detection [fast: true, auto-fix: false] + - forcetypeassert # finds forced type assertions [fast: true, auto-fix: false] + - gochecknoinits # Checks that no init functions are present in Go code [fast: true, auto-fix: false] + - goconst # Finds repeated strings that could be replaced by a constant [fast: true, auto-fix: false] + - gocritic # Provides diagnostics that check for bugs, performance and style issues. [fast: false, auto-fix: false] + - goimports # In addition to fixing imports, goimports also formats your code in the same style as gofmt. [fast: true, auto-fix: true] + - gomoddirectives # Manage the use of 'replace', 'retract', and 'excludes' directives in go.mod. [fast: true, auto-fix: false] + - gomodguard # Allow and block list linter for direct Go module dependencies. This is different from depguard where there are different block types for example version constraints and module recommendations. [fast: true, auto-fix: false] + - goprintffuncname # Checks that printf-like functions are named with `f` at the end [fast: true, auto-fix: false] + - gosec # (gas): Inspects source code for security problems [fast: false, auto-fix: false] + - grouper # An analyzer to analyze expression groups. [fast: true, auto-fix: false] + # - importas # Enforces consistent import aliases [fast: false, auto-fix: false] + - maintidx # maintidx measures the maintainability index of each function. [fast: true, auto-fix: false] + - makezero # Finds slice declarations with non-zero initial length [fast: false, auto-fix: false] + - misspell # Finds commonly misspelled English words in comments [fast: true, auto-fix: true] + - nakedret # Finds naked returns in functions greater than a specified function length [fast: true, auto-fix: false] + - nestif # Reports deeply nested if statements [fast: true, auto-fix: false] + - nilerr # Finds the code that returns nil even if it checks that the error is not nil. [fast: false, auto-fix: false] + - nilnil # Checks that there is no simultaneous return of `nil` error and an invalid value. [fast: false, auto-fix: false] + - noctx # noctx finds sending http request without context.Context [fast: false, auto-fix: false] + - nolintlint # Reports ill-formed or insufficient nolint directives [fast: true, auto-fix: false] + # - nonamedreturns # Reports all named returns [fast: false, auto-fix: false] + - nosprintfhostport # Checks for misuse of Sprintf to construct a host with port in a URL. [fast: true, auto-fix: false] + - prealloc # Finds slice declarations that could potentially be pre-allocated [fast: true, auto-fix: false] + - predeclared # find code that shadows one of Go's predeclared identifiers [fast: true, auto-fix: false] + - promlinter # Check Prometheus metrics naming via promlint [fast: true, auto-fix: false] + - revive # Fast, configurable, extensible, flexible, and beautiful linter for Go. Drop-in replacement of golint. [fast: false, auto-fix: false] + - tagliatelle # Checks the struct tags. [fast: true, auto-fix: false] + - testpackage # linter that makes you use a separate _test package [fast: true, auto-fix: false] + - thelper # thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers [fast: false, auto-fix: false] + - unconvert # Remove unnecessary type conversions [fast: false, auto-fix: false] + - unparam # Reports unused function parameters [fast: false, auto-fix: false] + - usestdlibvars # A linter that detect the possibility to use variables/constants from the Go standard library. [fast: true, auto-fix: false] + - wastedassign # wastedassign finds wasted assignment statements. [fast: false, auto-fix: false] + - whitespace # Tool for detection of leading and trailing whitespace [fast: true, auto-fix: true] + disable: + - unused diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..65027d3 --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,34 @@ +builds: + - skip: true + +changelog: + filters: + exclude: + - "^wip" + - "^test" + - "^docs" + - "^chore" + - "^style" + - "go mod tidy" + - "merge conflict" + - "Merge pull request" + - "Merge remote-tracking branch" + - "Merge branch" + groups: + - title: Features + regexp: '^.*?feat(\([[:word:]]+\))??!?:.+$' + order: 0 + - title: Dependency updates + regexp: '^.*?(feat|fix)\(deps\)!?:.+$' + order: 100 + - title: "Bug fixes" + regexp: '^.*?fix(\([[:word:]]+\))??!?:.+$' + order: 150 + - title: "Security" + regexp: '^.*?sec(\([[:word:]]+\))??!?:.+$' + order: 200 + - title: "Performace" + regexp: '^.*?perf(\([[:word:]]+\))??!?:.+$' + order: 250 + - title: Other + order: 999 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..26b0f06 --- /dev/null +++ b/Makefile @@ -0,0 +1,91 @@ +.DEFAULT_GOAL:=help +-include .makerc + +# --- Targets ----------------------------------------------------------------- + +# This allows us to accept extra arguments +%: .husky + @: + +.PHONY: .husky +# Configure git hooks for husky +.husky: + @if ! command -v husky &> /dev/null; then \ + echo "ERROR: missing executeable 'husky', please run:"; \ + echo "\n$ go install github.com/go-courier/husky/cmd/husky@latest\n"; \ + fi + @git config core.hooksPath .husky + +## === Tasks === + +.PHONY: doc +## Run tests +doc: + @open "http://localhost:6060/pkg/github.com/foomo/typesense/" + @godoc -http=localhost:6060 -play + +.PHONY: test +## Run tests +test: + @set -euo pipefail && go test -json -v ./... | gotestfmt + +.PHONY: test.cover +## Run tests with coverage +test.cover: + @go test -v -coverprofile=coverage.out ./... + @go tool cover -func=coverage.out + @go tool cover -html=coverage.out + +.PHONY: lint +## Run linter +lint: + @golangci-lint run + +.PHONY: lint.fix +## Run linter and fix violations +lint.fix: + @golangci-lint run --fix + +.PHONY: tidy +## Run go mod tidy +tidy: + @go mod tidy + +.PHONY: outdated +## Show outdated direct dependencies +outdated: + @go list -u -m -json all | go-mod-outdated -update -direct + +## === Utils === + +## Show help text +help: + @awk '{ \ + if ($$0 ~ /^.PHONY: [a-zA-Z\-\_0-9]+$$/) { \ + helpCommand = substr($$0, index($$0, ":") + 2); \ + if (helpMessage) { \ + printf "\033[36m%-23s\033[0m %s\n", \ + helpCommand, helpMessage; \ + helpMessage = ""; \ + } \ + } else if ($$0 ~ /^[a-zA-Z\-\_0-9.]+:/) { \ + helpCommand = substr($$0, 0, index($$0, ":")); \ + if (helpMessage) { \ + printf "\033[36m%-23s\033[0m %s\n", \ + helpCommand, helpMessage"\n"; \ + helpMessage = ""; \ + } \ + } else if ($$0 ~ /^##/) { \ + if (helpMessage) { \ + helpMessage = helpMessage"\n "substr($$0, 3); \ + } else { \ + helpMessage = substr($$0, 3); \ + } \ + } else { \ + if (helpMessage) { \ + print "\n "helpMessage"\n" \ + } \ + helpMessage = ""; \ + } \ + }' \ + $(MAKEFILE_LIST) diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..8423c46 --- /dev/null +++ b/go.mod @@ -0,0 +1,16 @@ +module github.com/foomo/typesense + +go 1.23.4 + +require ( + github.com/typesense/typesense-go/v3 v3.0.0 + go.uber.org/zap v1.27.0 +) + +require ( + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/oapi-codegen/runtime v1.1.1 // indirect + github.com/sony/gobreaker v1.0.0 // indirect + go.uber.org/multierr v1.10.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..59d0885 --- /dev/null +++ b/go.sum @@ -0,0 +1,35 @@ +github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= +github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jinzhu/copier v0.3.4 h1:mfU6jI9PtCeUjkjQ322dlff9ELjGDu975C2p/nrubVI= +github.com/jinzhu/copier v0.3.4/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= +github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= +github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sony/gobreaker v1.0.0 h1:feX5fGGXSl3dYd4aHZItw+FpHLvvoaqkawKjVNiFMNQ= +github.com/sony/gobreaker v1.0.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/typesense/typesense-go/v3 v3.0.0 h1:uLCMfVhv5GkZNjMGr/UHqVMKKdF0bkoTH4hAeigw8PE= +github.com/typesense/typesense-go/v3 v3.0.0/go.mod h1:Jx4PAXe3jRx6sc032nhN9Aj+OvMoPtQJW6p1a6H4Zeg= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= +go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/api/api.go b/pkg/api/api.go new file mode 100644 index 0000000..7d6a434 --- /dev/null +++ b/pkg/api/api.go @@ -0,0 +1,121 @@ +package typesenseapi + +import ( + "context" + "errors" + + "github.com/typesense/typesense-go/v3/typesense" + "go.uber.org/zap" + + "github.com/typesense/typesense-go/v3/typesense/api" +) + +const defaultSearchPresetName = "default" + +type BaseAPI[indexDocument any, returnDocument any] struct { + l *zap.Logger + client *typesense.Client + collections map[IndexID]*api.CollectionSchema + preset *api.PresetUpsertSchema + + revisionID RevisionID +} + +func NewBaseAPI[indexDocument any, returnDocument any]( + l *zap.Logger, + client *typesense.Client, + collections map[IndexID]*api.CollectionSchema, + preset *api.PresetUpsertSchema, +) *BaseAPI[indexDocument, returnDocument] { + return &BaseAPI[indexDocument, returnDocument]{ + l: l, + client: client, + collections: collections, + preset: preset, + } +} + +// Healthz will check if the revisionID is set +func (b *BaseAPI[indexDocument, returnDocument]) Healthz(_ context.Context) error { + if b.revisionID == "" { + return errors.New("revisionID not set") + } + return nil +} + +// Initialize +// will check the typesense connection and state of the colllections and aliases +// if the collections and aliases are not in the correct state it will create new collections and aliases +// +// example: +// +// b.collections := map[IndexID]*api.CollectionSchema{ +// "www-bks-at-de": { +// Name: "www-bks-at-de", +// }, +// "digital-bks-at-de": { +// Name: "digital-bks-at-de", +// } +// } +// +// there should be 2 aliases "www-bks-at-de" and "digital-bks-at-de" +// there should be at least 2 collections one for each alias +// the collection names are concatenated with the revisionID: "www-bks-at-de-2021-01-01-12" +// the revisionID is a timestamp in the format "YYYY-MM-DD-HH". If multiple collections are available +// the latest revisionID can be identified by the latest timestamp value +// +// Additionally, make sure that the configured search preset is present +// The system is ok if there is one alias for each collection and the collections are linked to the correct alias +// The function will set the revisionID that is currently linked to the aliases internally +func (b *BaseAPI[indexDocument, returnDocument]) Initialize() error { + var revisionID RevisionID + // use b.client.Health() to check the connection + + b.revisionID = revisionID + return nil +} + +func (b *BaseAPI[indexDocument, returnDocument]) NewRevision() (RevisionID, error) { + var revision RevisionID + + // create a revisionID based on the current time "YYYY-MM-DD-HH" + + // for all b.collections + // create a new collection in typesense - IndexID + - + revisionID + return revision, nil +} + +func (b *BaseAPI[indexDocument, returnDocument]) UpsertDocuments( + revisionID RevisionID, + indexID IndexID, + documents []indexDocument, +) error { + // use api to upsert documents + return nil +} + +// CommitRevision this is called when all the documents have been upserted +// it will update the aliases to point to the new revision +// additionally it will remove all old collections that are not linked to an alias +// keeping only the latest revision and the one before +func (b *BaseAPI[indexDocument, returnDocument]) CommitRevision(revisionID RevisionID) error { + return nil +} + +// SimpleSearch will perform a search operation on the given index +// it will return the documents and the scores +func (b *BaseAPI[indexDocument, returnDocument]) SimpleSearch( + index IndexID, + q string, + filterBy map[string]string, + page, perPage int, + sortBy string, +) ([]returnDocument, Scores, error) { + return b.ExpertSearch(index, getSearchCollectionParameters(q, filterBy, page, perPage, sortBy)) +} + +// ExpertSearch will perform a search operation on the given index +// it will return the documents and the scores +func (b *BaseAPI[indexDocument, returnDocument]) ExpertSearch(index IndexID, parameters *api.SearchCollectionParams) ([]returnDocument, Scores, error) { + return nil, nil, nil +} diff --git a/pkg/api/interface.go b/pkg/api/interface.go new file mode 100644 index 0000000..997a357 --- /dev/null +++ b/pkg/api/interface.go @@ -0,0 +1,24 @@ +package typesenseapi + +import "github.com/typesense/typesense-go/v3/typesense/api" + +type API[indexDocument any, returnDocument any] interface { + // this will prepare new indices with the given schema and the index IDs configured for the API + NewRevision() (RevisionID, error) + CommitRevision(revisionID RevisionID) error + UpsertDocuments(revisionID RevisionID, indexID IndexID, documents []indexDocument) error + + // this will check the typesense connection and initialize the indices + // should be run directly in a main.go or similar to ensure the connection is working + Initialize() (RevisionID, error) + + // perform a search operation on the given index + SimpleSearch( + index IndexID, + q string, + filterBy map[string]string, + page, perPage int, + sortBy string, + ) ([]returnDocument, Scores, error) + ExpertSearch(index IndexID, parameters *api.SearchCollectionParams) ([]returnDocument, Scores, error) +} diff --git a/pkg/api/type.go b/pkg/api/type.go new file mode 100644 index 0000000..be8be44 --- /dev/null +++ b/pkg/api/type.go @@ -0,0 +1,13 @@ +package typesenseapi + +type RevisionID string +type Query string +type IndexID string +type DocumentID string + +type Scores map[DocumentID]Score + +type Score struct { + ID DocumentID + Index int +} diff --git a/pkg/api/utils.go b/pkg/api/utils.go new file mode 100644 index 0000000..b985efa --- /dev/null +++ b/pkg/api/utils.go @@ -0,0 +1,42 @@ +package typesenseapi + +import ( + "strings" + + "github.com/typesense/typesense-go/v3/typesense/api" + "github.com/typesense/typesense-go/v3/typesense/api/pointer" +) + +// getSearchCollectionParameters will return the search collection parameters +// this is meant as a utility function to create the search collection parameters +// for the typesense search API without any knowledge of the typesense API +func getSearchCollectionParameters( + q string, + filterBy map[string]string, + page, perPage int, + sortBy string, +) *api.SearchCollectionParams { + parameters := &api.SearchCollectionParams{} + parameters.Q = pointer.String(q) + if filterByString := getFilterByString(filterBy); filterByString != "" { + parameters.FilterBy = pointer.String(filterByString) + } + parameters.Page = pointer.Int(page) + parameters.PerPage = pointer.Int(perPage) + if sortBy != "" { + parameters.SortBy = pointer.String(sortBy) + } + + return parameters +} + +func getFilterByString(filterBy map[string]string) string { + if filterBy == nil { + return "" + } + filterByString := []string{} + for key, value := range filterBy { + filterByString = append(filterByString, key+":="+value) + } + return strings.Join(filterByString, "&&") +}