diff --git a/go.mod b/go.mod index 8423c46..6b1d7f6 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,44 @@ module github.com/foomo/typesense go 1.23.4 require ( + github.com/foomo/contentserver v1.11.2 github.com/typesense/typesense-go/v3 v3.0.0 go.uber.org/zap v1.27.0 ) require ( github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect + github.com/avast/retry-go/v4 v4.6.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/fbiville/markdown-table-formatter v0.3.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/foomo/gostandards v0.2.0 // indirect + github.com/foomo/keel v0.19.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oapi-codegen/runtime v1.1.1 // indirect + github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.20.5 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.60.1 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/sony/gobreaker v1.0.0 // indirect - go.uber.org/multierr v1.10.0 // indirect + github.com/tinylib/msgp v1.2.4 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 // indirect + go.opentelemetry.io/otel v1.32.0 // indirect + go.opentelemetry.io/otel/metric v1.32.0 // indirect + go.opentelemetry.io/otel/trace v1.32.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.29.0 // indirect + golang.org/x/sync v0.9.0 // indirect + golang.org/x/sys v0.27.0 // indirect + google.golang.org/protobuf v1.35.1 // indirect ) diff --git a/go.sum b/go.sum index 59d0885..bb52e49 100644 --- a/go.sum +++ b/go.sum @@ -1,35 +1,104 @@ github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= +github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA= +github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fbiville/markdown-table-formatter v0.3.0 h1:PIm1UNgJrFs8q1htGTw+wnnNYvwXQMMMIKNZop2SSho= +github.com/fbiville/markdown-table-formatter v0.3.0/go.mod h1:q89TDtSEVDdTaufgSbfHpNVdPU/bmfvqNkrC5HagmLY= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/foomo/contentserver v1.11.2 h1:q5akkCoWS8e0vq+N2gIbmCjKuSCuzVJTeQ1V2JIqD+U= +github.com/foomo/contentserver v1.11.2/go.mod h1:shgTH0C6sUJAA/4qZvPprWf3VnsJ97XjVxO7ctw+q+A= +github.com/foomo/gostandards v0.2.0 h1:Ryd7TI9yV3Xk5B84DcUDB7KcL3LzQ8NS+TVOrFxTYfA= +github.com/foomo/gostandards v0.2.0/go.mod h1:XQx7Ur6vyvxaIe2cQvAthuhPYDe+d2soibqVcXDXOh4= +github.com/foomo/keel v0.19.0 h1:8uIinFat9Jj72zyWx6c+30f2o0EdXZ350s/caEC37P8= +github.com/foomo/keel v0.19.0/go.mod h1:eyO1lVDIvuIOFjWdIx5MqnWmk0E0FZWZwFhtLiVkTio= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jinzhu/copier v0.3.4 h1:mfU6jI9PtCeUjkjQ322dlff9ELjGDu975C2p/nrubVI= github.com/jinzhu/copier v0.3.4/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/oapi-codegen/runtime v1.1.1 h1:EXLHh0DXIJnWhdRPN2w4MXAzFyE4CskzhNLUmtpMYro= github.com/oapi-codegen/runtime v1.1.1/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c h1:dAMKvw0MlJT1GshSTtih8C2gDs04w8dReiOGXrGLNoY= +github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc= +github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/sony/gobreaker v1.0.0 h1:feX5fGGXSl3dYd4aHZItw+FpHLvvoaqkawKjVNiFMNQ= github.com/sony/gobreaker v1.0.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tinylib/msgp v1.2.4 h1:yLFeUGostXXSGW5vxfT5dXG/qzkn4schv2I7at5+hVU= +github.com/tinylib/msgp v1.2.4/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= github.com/typesense/typesense-go/v3 v3.0.0 h1:uLCMfVhv5GkZNjMGr/UHqVMKKdF0bkoTH4hAeigw8PE= github.com/typesense/typesense-go/v3 v3.0.0/go.mod h1:Jx4PAXe3jRx6sc032nhN9Aj+OvMoPtQJW6p1a6H4Zeg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 h1:DheMAlT6POBP+gh8RUH19EOTnQIor5QE0uSRPtzCpSw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0/go.mod h1:wZcGmeVO9nzP67aYSLDqXNWK87EZWhi7JWj1v7ZXf94= +go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= +go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= +go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= +go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= +go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= +go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= -go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= -go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= +golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= +golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= +golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= +golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/api/api.go b/pkg/api/api.go index 7d6a434..78edeef 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -3,31 +3,30 @@ package typesenseapi import ( "context" "errors" - + typesense2 "github.com/foomo/typesense/pkg" "github.com/typesense/typesense-go/v3/typesense" - "go.uber.org/zap" - "github.com/typesense/typesense-go/v3/typesense/api" + "go.uber.org/zap" ) const defaultSearchPresetName = "default" -type BaseAPI[indexDocument any, returnDocument any] struct { +type BaseAPI[indexDocument any, returnType any] struct { l *zap.Logger client *typesense.Client - collections map[IndexID]*api.CollectionSchema + collections map[typesense2.IndexID]*api.CollectionSchema preset *api.PresetUpsertSchema - revisionID RevisionID + revisionID typesense2.RevisionID } -func NewBaseAPI[indexDocument any, returnDocument any]( +func NewBaseAPI[indexDocument any, returnType any]( l *zap.Logger, client *typesense.Client, - collections map[IndexID]*api.CollectionSchema, + collections map[typesense2.IndexID]*api.CollectionSchema, preset *api.PresetUpsertSchema, -) *BaseAPI[indexDocument, returnDocument] { - return &BaseAPI[indexDocument, returnDocument]{ +) *BaseAPI[indexDocument, returnType] { + return &BaseAPI[indexDocument, returnType]{ l: l, client: client, collections: collections, @@ -36,13 +35,25 @@ func NewBaseAPI[indexDocument any, returnDocument any]( } // Healthz will check if the revisionID is set -func (b *BaseAPI[indexDocument, returnDocument]) Healthz(_ context.Context) error { +func (b *BaseAPI[indexDocument, returnType]) Healthz(_ context.Context) error { if b.revisionID == "" { return errors.New("revisionID not set") } return nil } +// Healthz will check if the revisionID is set +func (b *BaseAPI[indexDocument, returnType]) Indices() ([]typesense2.IndexID, error) { + if len(b.collections) == 0 { + return nil, errors.New("no collections configured") + } + indices := make([]typesense2.IndexID, 0, len(b.collections)) + for index := range b.collections { + indices = append(indices, index) + } + return indices, nil +} + // Initialize // will check the typesense connection and state of the colllections and aliases // if the collections and aliases are not in the correct state it will create new collections and aliases @@ -67,16 +78,16 @@ func (b *BaseAPI[indexDocument, returnDocument]) Healthz(_ context.Context) erro // Additionally, make sure that the configured search preset is present // The system is ok if there is one alias for each collection and the collections are linked to the correct alias // The function will set the revisionID that is currently linked to the aliases internally -func (b *BaseAPI[indexDocument, returnDocument]) Initialize() error { - var revisionID RevisionID +func (b *BaseAPI[indexDocument, returnType]) Initialize() (typesense2.RevisionID, error) { + var revisionID typesense2.RevisionID // use b.client.Health() to check the connection b.revisionID = revisionID - return nil + return "", nil } -func (b *BaseAPI[indexDocument, returnDocument]) NewRevision() (RevisionID, error) { - var revision RevisionID +func (b *BaseAPI[indexDocument, returnType]) NewRevision() (typesense2.RevisionID, error) { + var revision typesense2.RevisionID // create a revisionID based on the current time "YYYY-MM-DD-HH" @@ -85,9 +96,9 @@ func (b *BaseAPI[indexDocument, returnDocument]) NewRevision() (RevisionID, erro return revision, nil } -func (b *BaseAPI[indexDocument, returnDocument]) UpsertDocuments( - revisionID RevisionID, - indexID IndexID, +func (b *BaseAPI[indexDocument, returnType]) UpsertDocuments( + revisionID typesense2.RevisionID, + indexID typesense2.IndexID, documents []indexDocument, ) error { // use api to upsert documents @@ -98,24 +109,29 @@ func (b *BaseAPI[indexDocument, returnDocument]) UpsertDocuments( // it will update the aliases to point to the new revision // additionally it will remove all old collections that are not linked to an alias // keeping only the latest revision and the one before -func (b *BaseAPI[indexDocument, returnDocument]) CommitRevision(revisionID RevisionID) error { +func (b *BaseAPI[indexDocument, returnType]) CommitRevision(revisionID typesense2.RevisionID) error { + return nil +} + +// RevertRevision will remove the collections created for the given revisionID +func (b *BaseAPI[indexDocument, returnType]) RevertRevision(revisionID typesense2.RevisionID) error { return nil } // SimpleSearch will perform a search operation on the given index // it will return the documents and the scores -func (b *BaseAPI[indexDocument, returnDocument]) SimpleSearch( - index IndexID, +func (b *BaseAPI[indexDocument, returnType]) SimpleSearch( + index typesense2.IndexID, q string, filterBy map[string]string, page, perPage int, sortBy string, -) ([]returnDocument, Scores, error) { +) ([]returnType, typesense2.Scores, error) { return b.ExpertSearch(index, getSearchCollectionParameters(q, filterBy, page, perPage, sortBy)) } // ExpertSearch will perform a search operation on the given index // it will return the documents and the scores -func (b *BaseAPI[indexDocument, returnDocument]) ExpertSearch(index IndexID, parameters *api.SearchCollectionParams) ([]returnDocument, Scores, error) { +func (b *BaseAPI[indexDocument, returnType]) ExpertSearch(index typesense2.IndexID, parameters *api.SearchCollectionParams) ([]returnType, typesense2.Scores, error) { return nil, nil, nil } diff --git a/pkg/api/type.go b/pkg/api/type.go deleted file mode 100644 index be8be44..0000000 --- a/pkg/api/type.go +++ /dev/null @@ -1,13 +0,0 @@ -package typesenseapi - -type RevisionID string -type Query string -type IndexID string -type DocumentID string - -type Scores map[DocumentID]Score - -type Score struct { - ID DocumentID - Index int -} diff --git a/pkg/indexing/contentserver.go b/pkg/indexing/contentserver.go new file mode 100644 index 0000000..7e27884 --- /dev/null +++ b/pkg/indexing/contentserver.go @@ -0,0 +1,107 @@ +package typesenseindexing + +import ( + "context" + "fmt" + "github.com/foomo/contentserver/client" + "github.com/foomo/contentserver/content" + typesense "github.com/foomo/typesense/pkg" + "go.uber.org/zap" +) + +type ContentServer[indexDocument any] struct { + l *zap.Logger + client *client.Client + documentProviderFuncs map[typesense.DocumentType]typesense.DocumentProviderFunc[indexDocument] +} + +func NewContentServer[indexDocument any]( + l *zap.Logger, + client *client.Client, + documentProviderFuncs map[typesense.DocumentType]typesense.DocumentProviderFunc[indexDocument], +) *ContentServer[indexDocument] { + return &ContentServer[indexDocument]{ + l: l, + client: client, + documentProviderFuncs: documentProviderFuncs, + } +} + +func (c ContentServer[indexDocument]) Provide( + ctx context.Context, + indexID typesense.IndexID, +) ([]indexDocument, error) { + documentInfos, err := c.getDocumentIDsByIndexID(ctx, indexID) + if err != nil { + return nil, err + } + documents := make([]indexDocument, len(documentInfos)) + for index, documentInfo := range documentInfos { + if documentProvider, ok := c.documentProviderFuncs[documentInfo.DocumentType]; !ok { + c.l.Warn("no document provider available for document type", zap.String("documentType", string(documentInfo.DocumentType))) + } else { + document, err := documentProvider(ctx, indexID, documentInfo.DocumentID) + if err != nil { + c.l.Error( + "index document not created", + zap.Error(err), + zap.String("documentID", string(documentInfo.DocumentID)), + zap.String("documentType", string(documentInfo.DocumentType)), + ) + continue + } + documents[index] = document + } + } + return documents, nil +} + +func (c ContentServer[indexDocument]) ProvidePaged( + ctx context.Context, + indexID typesense.IndexID, + offset int, +) ([]indexDocument, int, error) { + panic("implement me") + return nil, 0, nil +} + +func (c ContentServer[indexDocument]) getDocumentIDsByIndexID( + ctx context.Context, + indexID typesense.IndexID, +) ([]typesense.DocumentInfo, error) { + // get the contentserver dimension defined by indexID + // create the list of document infos + repo, err := c.client.GetRepo(ctx) + if err != nil { + return nil, err + } + rootRepoNode, ok := repo[string(indexID)] + if !ok { + return nil, fmt.Errorf("contenserver dimension %s not found", indexID) + } + + nodeMap := createFlatRepoNodeMap(rootRepoNode, map[string]*content.RepoNode{}) + documentInfos := make([]typesense.DocumentInfo, 0, len(nodeMap)) + for _, repoNode := range nodeMap { + documentInfos = append(documentInfos, typesense.DocumentInfo{ + DocumentType: typesense.DocumentType(repoNode.MimeType), + DocumentID: typesense.DocumentID(repoNode.ID), + }) + } + + return documentInfos, nil +} + +// createFlatRepoNodeMap recursively retrieves all nodes from the tree and returns them in a flat map. +func createFlatRepoNodeMap(node *content.RepoNode, nodeMap map[string]*content.RepoNode) map[string]*content.RepoNode { + if node == nil { + return nodeMap + } + // Add the current node to the list. + nodeMap[node.ID] = node + // Recursively process child nodes. + for _, child := range node.Nodes { + nodeMap = createFlatRepoNodeMap(child, nodeMap) + } + return nodeMap +} diff --git a/pkg/indexing/indexer.go b/pkg/indexing/indexer.go new file mode 100644 index 0000000..af76883 --- /dev/null +++ b/pkg/indexing/indexer.go @@ -0,0 +1,88 @@ +package typesenseindexing + +import ( + "context" + typesense "github.com/foomo/typesense/pkg" + "go.uber.org/zap" +) + +type BaseIndexer[indexDocument any, returnType any] struct { + l *zap.Logger + typesenseAPI typesense.API[indexDocument, returnType] + documentProvider typesense.DocumentProvider[indexDocument] +} + +func NewBaseIndexer[indexDocument any, returnType any]( + l *zap.Logger, + typesenseAPI typesense.API[indexDocument, returnType], + documentProvider typesense.DocumentProvider[indexDocument], +) *BaseIndexer[indexDocument, returnType] { + return &BaseIndexer[indexDocument, returnType]{ + l: l, + typesenseAPI: typesenseAPI, + documentProvider: documentProvider, + } +} + +func (b *BaseIndexer[indexDocument, returnType]) Healthz(ctx context.Context) error { + return b.typesenseAPI.Healthz(ctx) +} + +func (b *BaseIndexer[indexDocument, returnType]) Run(ctx context.Context) error { + // return error if the health check fails + if err := b.Healthz(ctx); err != nil { + return err + } + + // create a new revision + revisionID, err := b.typesenseAPI.NewRevision() + if err != nil { + return err + } + + // get the configured indices from the typesense API + indices, err := b.typesenseAPI.Indices() + if err != nil { + return err + } + + // set a variable to check if the upserting of documents was successful + tainted := false + + // for each index, get the documents from the document provider and upsert them + for _, indexID := range indices { + documents, err := b.documentProvider.Provide(ctx, indexID) + if err != nil { + return err + } + + err = b.typesenseAPI.UpsertDocuments(revisionID, indexID, documents) + if err != nil { + b.l.Error( + "failed to upsert documents", + zap.Error(err), + zap.String("index", string(indexID)), + zap.String("revision", string(revisionID)), + zap.Int("documents", len(documents)), + ) + tainted = true + break + } + } + + if !tainted { + // commit the revision if no errors occurred + err = b.typesenseAPI.CommitRevision(revisionID) + if err != nil { + return err + } + } else { + // revert the revision if errors occurred + err = b.typesenseAPI.RevertRevision(revisionID) + if err != nil { + return err + } + } + + return nil +} diff --git a/pkg/api/interface.go b/pkg/interface.go similarity index 51% rename from pkg/api/interface.go rename to pkg/interface.go index 997a357..2cf1cf5 100644 --- a/pkg/api/interface.go +++ b/pkg/interface.go @@ -1,11 +1,15 @@ -package typesenseapi +package typesense -import "github.com/typesense/typesense-go/v3/typesense/api" +import ( + "context" + "github.com/typesense/typesense-go/v3/typesense/api" +) -type API[indexDocument any, returnDocument any] interface { +type API[indexDocument any, returnType any] interface { // this will prepare new indices with the given schema and the index IDs configured for the API NewRevision() (RevisionID, error) CommitRevision(revisionID RevisionID) error + RevertRevision(revisionID RevisionID) error UpsertDocuments(revisionID RevisionID, indexID IndexID, documents []indexDocument) error // this will check the typesense connection and initialize the indices @@ -19,6 +23,17 @@ type API[indexDocument any, returnDocument any] interface { filterBy map[string]string, page, perPage int, sortBy string, - ) ([]returnDocument, Scores, error) - ExpertSearch(index IndexID, parameters *api.SearchCollectionParams) ([]returnDocument, Scores, error) + ) ([]returnType, Scores, error) + ExpertSearch(index IndexID, parameters *api.SearchCollectionParams) ([]returnType, Scores, error) + Healthz(ctx context.Context) error + Indices() ([]IndexID, error) +} + +type IndexerInterface[indexDocument any, returnType any] interface { + Run(ctx context.Context) error +} + +type DocumentProvider[indexDocument any] interface { + Provide(ctx context.Context, index IndexID) ([]indexDocument, error) + ProvidePaged(ctx context.Context, index IndexID, offset int) ([]indexDocument, int, error) } diff --git a/pkg/vo.go b/pkg/vo.go new file mode 100644 index 0000000..8f485ae --- /dev/null +++ b/pkg/vo.go @@ -0,0 +1,27 @@ +package typesense + +import "context" + +type RevisionID string +type Query string +type IndexID string +type DocumentID string +type DocumentType string + +type Scores map[DocumentID]Score + +type Score struct { + ID DocumentID + Index int +} + +type DocumentProviderFunc[indexDocument any] func( + ctx context.Context, + indexID IndexID, + documentID DocumentID, +) (indexDocument, error) + +type DocumentInfo struct { + DocumentType DocumentType + DocumentID DocumentID +}