From 6a1a8e796f4371168cff8921bdabbcc52c91b973 Mon Sep 17 00:00:00 2001 From: Miroslav Cvetic Date: Tue, 4 Mar 2025 14:07:49 +0100 Subject: [PATCH 1/8] feat: support filter by multiple values --- pkg/api/api.go | 2 +- pkg/api/utils.go | 34 ++++++++++++++++++---------------- pkg/interface.go | 2 +- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index 6617fc2..faaa021 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -270,7 +270,7 @@ func (b *BaseAPI[indexDocument, returnType]) SimpleSearch( ctx context.Context, index pkgtypesense.IndexID, q string, - filterBy map[string]string, + filterBy map[string][]string, page, perPage int, sortBy string, ) ([]returnType, pkgtypesense.Scores, error) { diff --git a/pkg/api/utils.go b/pkg/api/utils.go index 7a3a504..2efabeb 100644 --- a/pkg/api/utils.go +++ b/pkg/api/utils.go @@ -18,7 +18,7 @@ import ( // for the typesense search API without any knowledge of the typesense API func buildSearchParams( q string, - filterBy map[string]string, + filterBy map[string][]string, // Updated to allow multiple values per field page, perPage int, sortBy string, ) *api.SearchCollectionParams { @@ -36,31 +36,33 @@ func buildSearchParams( return parameters } -func formatFilterQuery(filterBy map[string]string) string { +func formatFilterQuery(filterBy map[string][]string) string { if filterBy == nil { return "" } - filterByString := []string{} - for key, value := range filterBy { - filterByString = append(filterByString, key+":="+value) + + var filterClauses []string + for key, values := range filterBy { + if len(values) == 1 { + // Single value → Use `:=` operator + filterClauses = append(filterClauses, fmt.Sprintf("%s:=\"%s\"", key, values[0])) + } else { + // Multiple values → Use `["val1","val2"]` array syntax + formattedValues := []string{} + for _, v := range values { + formattedValues = append(formattedValues, fmt.Sprintf("\"%s\"", v)) + } + filterClauses = append(filterClauses, fmt.Sprintf("%s:[%s]", key, strings.Join(formattedValues, ","))) + } } - return strings.Join(filterByString, "&&") + + return strings.Join(filterClauses, " && ") // AND conditions by default } func (b *BaseAPI[indexDocument, returnType]) generateRevisionID() pkgtypesense.RevisionID { return pkgtypesense.RevisionID(time.Now().Format("2006-01-02-15-04")) // "YYYY-MM-DD-HH-MM" } -func getLatestRevisionID(revisions map[pkgtypesense.IndexID]pkgtypesense.RevisionID) pkgtypesense.RevisionID { - var latest pkgtypesense.RevisionID - for _, rev := range revisions { - if rev > latest { - latest = rev - } - } - return latest -} - func formatCollectionName(indexID pkgtypesense.IndexID, revisionID pkgtypesense.RevisionID) string { return fmt.Sprintf("%s-%s", indexID, revisionID) } diff --git a/pkg/interface.go b/pkg/interface.go index 3865237..7573f87 100644 --- a/pkg/interface.go +++ b/pkg/interface.go @@ -21,7 +21,7 @@ type API[indexDocument any, returnType any] interface { ctx context.Context, index IndexID, q string, - filterBy map[string]string, + filterBy map[string][]string, page, perPage int, sortBy string, ) ([]returnType, Scores, error) From c3c0fe35a23233d7a5e1d7664ef70f943582b050 Mon Sep 17 00:00:00 2001 From: Miroslav Cvetic Date: Tue, 4 Mar 2025 15:25:24 +0100 Subject: [PATCH 2/8] feature: add total results to search response --- pkg/api/api.go | 24 +++++++++++++++++------- pkg/interface.go | 4 ++-- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index faaa021..a4703e7 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -273,7 +273,7 @@ func (b *BaseAPI[indexDocument, returnType]) SimpleSearch( filterBy map[string][]string, page, perPage int, sortBy string, -) ([]returnType, pkgtypesense.Scores, error) { +) ([]returnType, pkgtypesense.Scores, int, error) { // Call buildSearchParams but also set QueryBy explicitly parameters := buildSearchParams(q, filterBy, page, perPage, sortBy) parameters.QueryBy = pointer.String("title") @@ -282,26 +282,35 @@ func (b *BaseAPI[indexDocument, returnType]) SimpleSearch( } // ExpertSearch will perform a search operation on the given index -// it will return the documents and the scores +// it will return the documents, scores, and totalResults func (b *BaseAPI[indexDocument, returnType]) ExpertSearch( ctx context.Context, indexID pkgtypesense.IndexID, parameters *api.SearchCollectionParams, -) ([]returnType, pkgtypesense.Scores, error) { +) ([]returnType, pkgtypesense.Scores, int, error) { if parameters == nil { b.l.Error("Search parameters are nil") - return nil, nil, errors.New("search parameters cannot be nil") + return nil, nil, 0, errors.New("search parameters cannot be nil") } collectionName := string(indexID) // digital-bks-at-de searchResponse, err := b.client.Collection(collectionName).Documents().Search(ctx, parameters) if err != nil { b.l.Error("Failed to perform search", zap.String("index", collectionName), zap.Error(err)) - return nil, nil, err + return nil, nil, 0, err } + // Handle nil response + if searchResponse == nil || searchResponse.Hits == nil { + b.l.Warn("Search response or hits is nil", zap.String("index", collectionName)) + return nil, nil, 0, nil + } + + // Extract totalResults from the search response + totalResults := *searchResponse.Found + // Parse search results - var results = make([]returnType, 0, len(*searchResponse.Hits)) + results := make([]returnType, 0, len(*searchResponse.Hits)) scores := make(pkgtypesense.Scores) for _, hit := range *searchResponse.Hits { @@ -341,7 +350,8 @@ func (b *BaseAPI[indexDocument, returnType]) ExpertSearch( b.l.Info("Search completed", zap.String("index", collectionName), zap.Int("results_count", len(results)), + zap.Int("total_results", totalResults), ) - return results, scores, nil + return results, scores, totalResults, nil } diff --git a/pkg/interface.go b/pkg/interface.go index 7573f87..b838883 100644 --- a/pkg/interface.go +++ b/pkg/interface.go @@ -24,8 +24,8 @@ type API[indexDocument any, returnType any] interface { filterBy map[string][]string, page, perPage int, sortBy string, - ) ([]returnType, Scores, error) - ExpertSearch(ctx context.Context, index IndexID, parameters *api.SearchCollectionParams) ([]returnType, Scores, error) + ) ([]returnType, Scores, int, error) + ExpertSearch(ctx context.Context, index IndexID, parameters *api.SearchCollectionParams) ([]returnType, Scores, int, error) Healthz(ctx context.Context) error Indices() ([]IndexID, error) } From 9f858f42ea8c1aced471471a968cd6ae3982816d Mon Sep 17 00:00:00 2001 From: Miroslav Cvetic Date: Tue, 4 Mar 2025 16:51:11 +0100 Subject: [PATCH 3/8] feat: pass uriMap to documentProvider --- pkg/indexing/contentserver.go | 13 ++++++++++++- pkg/vo.go | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/indexing/contentserver.go b/pkg/indexing/contentserver.go index fe99311..84ec0b2 100644 --- a/pkg/indexing/contentserver.go +++ b/pkg/indexing/contentserver.go @@ -40,12 +40,23 @@ func (c ContentServer[indexDocument]) Provide( if err != nil { return nil, err } + + ids := make([]string, 0, len(documentInfos)) + for _, documentInfo := range documentInfos { + ids = append(ids, string(documentInfo.DocumentID)) + } + + uriMap, err := c.contentserverClient.GetURIs(ctx, string(indexID), ids) + if err != nil { + return nil, err + } + documents := make([]*indexDocument, len(documentInfos)) for index, documentInfo := range documentInfos { if documentProvider, ok := c.documentProviderFuncs[documentInfo.DocumentType]; !ok { c.l.Warn("no document provider available for document type", zap.String("documentType", string(documentInfo.DocumentType))) } else { - document, err := documentProvider(ctx, indexID, documentInfo.DocumentID) + document, err := documentProvider(ctx, indexID, documentInfo.DocumentID, uriMap) if err != nil { c.l.Error( "index document not created", diff --git a/pkg/vo.go b/pkg/vo.go index 1783932..3805bbd 100644 --- a/pkg/vo.go +++ b/pkg/vo.go @@ -19,6 +19,7 @@ type DocumentProviderFunc[indexDocument any] func( ctx context.Context, indexID IndexID, documentID DocumentID, + uriMap map[string]string, ) (*indexDocument, error) type DocumentInfo struct { From 8b2b415ae3a8626f04287865446e08b93e4132ea Mon Sep 17 00:00:00 2001 From: Miroslav Cvetic Date: Tue, 4 Mar 2025 17:01:02 +0100 Subject: [PATCH 4/8] feat: go version --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 6b1d7f6..a56b88d 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/foomo/typesense -go 1.23.4 +go 1.24 require ( github.com/foomo/contentserver v1.11.2 From 64cea36c6acaf42c45b54dce4014492f1e1fb267 Mon Sep 17 00:00:00 2001 From: Miroslav Cvetic Date: Tue, 4 Mar 2025 17:15:08 +0100 Subject: [PATCH 5/8] feat: go version --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a56b88d..c33b95b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/foomo/typesense -go 1.24 +go 1.24.0 require ( github.com/foomo/contentserver v1.11.2 From 51e1dca8787e5f47f5a1bed8eb0b7e82d26959db Mon Sep 17 00:00:00 2001 From: Miroslav Cvetic Date: Thu, 6 Mar 2025 09:55:45 +0100 Subject: [PATCH 6/8] feat: PR prepare --- pkg/api/utils.go | 4 ++-- pkg/indexing/contentserver.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/api/utils.go b/pkg/api/utils.go index 2efabeb..818f849 100644 --- a/pkg/api/utils.go +++ b/pkg/api/utils.go @@ -18,7 +18,7 @@ import ( // for the typesense search API without any knowledge of the typesense API func buildSearchParams( q string, - filterBy map[string][]string, // Updated to allow multiple values per field + filterBy map[string][]string, page, perPage int, sortBy string, ) *api.SearchCollectionParams { @@ -56,7 +56,7 @@ func formatFilterQuery(filterBy map[string][]string) string { } } - return strings.Join(filterClauses, " && ") // AND conditions by default + return strings.Join(filterClauses, " && ") } func (b *BaseAPI[indexDocument, returnType]) generateRevisionID() pkgtypesense.RevisionID { diff --git a/pkg/indexing/contentserver.go b/pkg/indexing/contentserver.go index 84ec0b2..4128ad2 100644 --- a/pkg/indexing/contentserver.go +++ b/pkg/indexing/contentserver.go @@ -48,6 +48,7 @@ func (c ContentServer[indexDocument]) Provide( uriMap, err := c.contentserverClient.GetURIs(ctx, string(indexID), ids) if err != nil { + c.l.Error("failed to get URIs", zap.Error(err)) return nil, err } From 7ba353715a3dcbef62cf5af36279a537411781c6 Mon Sep 17 00:00:00 2001 From: Miroslav Cvetic Date: Thu, 6 Mar 2025 15:45:08 +0100 Subject: [PATCH 7/8] pr fixes --- pkg/api/api.go | 4 ++-- pkg/indexing/contentserver.go | 38 +++++++++++++++++++++++++++-------- pkg/vo.go | 2 +- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index a4703e7..7cc63d6 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -313,7 +313,7 @@ func (b *BaseAPI[indexDocument, returnType]) ExpertSearch( results := make([]returnType, 0, len(*searchResponse.Hits)) scores := make(pkgtypesense.Scores) - for _, hit := range *searchResponse.Hits { + for i, hit := range *searchResponse.Hits { docMap := *hit.Document // Extract document ID safely @@ -331,7 +331,7 @@ func (b *BaseAPI[indexDocument, returnType]) ExpertSearch( continue } - results = append(results, doc) + results[i] = doc index := 0 if hit.TextMatchInfo != nil && hit.TextMatchInfo.Score != nil { if score, err := strconv.Atoi(*hit.TextMatchInfo.Score); err == nil { diff --git a/pkg/indexing/contentserver.go b/pkg/indexing/contentserver.go index 4128ad2..58eeb28 100644 --- a/pkg/indexing/contentserver.go +++ b/pkg/indexing/contentserver.go @@ -41,14 +41,8 @@ func (c ContentServer[indexDocument]) Provide( return nil, err } - ids := make([]string, 0, len(documentInfos)) - for _, documentInfo := range documentInfos { - ids = append(ids, string(documentInfo.DocumentID)) - } - - uriMap, err := c.contentserverClient.GetURIs(ctx, string(indexID), ids) + urlsByIDs, err := c.fetchURLsByDocumentIDs(ctx, indexID, documentInfos) if err != nil { - c.l.Error("failed to get URIs", zap.Error(err)) return nil, err } @@ -57,7 +51,7 @@ func (c ContentServer[indexDocument]) Provide( if documentProvider, ok := c.documentProviderFuncs[documentInfo.DocumentType]; !ok { c.l.Warn("no document provider available for document type", zap.String("documentType", string(documentInfo.DocumentType))) } else { - document, err := documentProvider(ctx, indexID, documentInfo.DocumentID, uriMap) + document, err := documentProvider(ctx, indexID, documentInfo.DocumentID, urlsByIDs) if err != nil { c.l.Error( "index document not created", @@ -125,3 +119,31 @@ func createFlatRepoNodeMap(node *content.RepoNode, nodeMap map[string]*content.R } return nodeMap } + +func (c ContentServer[indexDocument]) fetchURLsByDocumentIDs( + ctx context.Context, + indexID typesense.IndexID, + documentInfos []typesense.DocumentInfo, +) (map[typesense.DocumentID]string, error) { + ids := make([]string, len(documentInfos)) + + for i, documentInfo := range documentInfos { + ids[i] = string(documentInfo.DocumentID) + } + + uriMap, err := c.contentserverClient.GetURIs(ctx, string(indexID), ids) + if err != nil { + c.l.Error("failed to get URIs", zap.Error(err)) + return nil, err + } + + return convertMapStringToDocumentID(uriMap), nil +} + +func convertMapStringToDocumentID(input map[string]string) map[typesense.DocumentID]string { + output := make(map[typesense.DocumentID]string, len(input)) + for key, value := range input { + output[typesense.DocumentID(key)] = value + } + return output +} diff --git a/pkg/vo.go b/pkg/vo.go index 3805bbd..b7acbb2 100644 --- a/pkg/vo.go +++ b/pkg/vo.go @@ -19,7 +19,7 @@ type DocumentProviderFunc[indexDocument any] func( ctx context.Context, indexID IndexID, documentID DocumentID, - uriMap map[string]string, + urlsByIDs map[DocumentID]string, ) (*indexDocument, error) type DocumentInfo struct { From 4a129315c1840ecfe57e8eed0cfcee27813f957c Mon Sep 17 00:00:00 2001 From: Miroslav Cvetic Date: Thu, 6 Mar 2025 16:15:37 +0100 Subject: [PATCH 8/8] ensure safe indexing --- pkg/api/api.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index 7cc63d6..fef178c 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -299,21 +299,24 @@ func (b *BaseAPI[indexDocument, returnType]) ExpertSearch( b.l.Error("Failed to perform search", zap.String("index", collectionName), zap.Error(err)) return nil, nil, 0, err } - - // Handle nil response - if searchResponse == nil || searchResponse.Hits == nil { - b.l.Warn("Search response or hits is nil", zap.String("index", collectionName)) - return nil, nil, 0, nil - } - // Extract totalResults from the search response totalResults := *searchResponse.Found - // Parse search results - results := make([]returnType, 0, len(*searchResponse.Hits)) + // Ensure Hits is not empty before proceeding + if searchResponse.Hits == nil || len(*searchResponse.Hits) == 0 { + b.l.Warn("Search response contains no hits", zap.String("index", collectionName)) + return nil, nil, totalResults, nil + } + + results := make([]returnType, len(*searchResponse.Hits)) scores := make(pkgtypesense.Scores) for i, hit := range *searchResponse.Hits { + if hit.Document == nil { + b.l.Warn("Hit document is nil", zap.String("index", collectionName)) + continue + } + docMap := *hit.Document // Extract document ID safely