From e979d3076d944cfdf9dbe4823e15b0f8ad119921 Mon Sep 17 00:00:00 2001 From: moshaad7 Date: Mon, 8 Jul 2024 01:40:25 +0530 Subject: [PATCH 1/3] MB-61889: support search with params --- faiss_vector_posting.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/faiss_vector_posting.go b/faiss_vector_posting.go index 9d1fe171..e4275d76 100644 --- a/faiss_vector_posting.go +++ b/faiss_vector_posting.go @@ -19,6 +19,7 @@ package zap import ( "encoding/binary" + "encoding/json" "math" "reflect" @@ -266,14 +267,14 @@ func (vpl *VecPostingsIterator) BytesWritten() uint64 { // vectorIndexWrapper conforms to scorch_segment_api's VectorIndex interface type vectorIndexWrapper struct { - search func(qVector []float32, k int64) (segment.VecPostingsList, error) + search func(qVector []float32, k int64, params json.RawMessage) (segment.VecPostingsList, error) close func() size func() uint64 } -func (i *vectorIndexWrapper) Search(qVector []float32, k int64) ( +func (i *vectorIndexWrapper) Search(qVector []float32, k int64, params json.RawMessage) ( segment.VecPostingsList, error) { - return i.search(qVector, k) + return i.search(qVector, k, params) } func (i *vectorIndexWrapper) Close() { @@ -300,7 +301,7 @@ func (sb *SegmentBase) InterpretVectorIndex(field string, except *roaring.Bitmap var ( wrapVecIndex = &vectorIndexWrapper{ - search: func(qVector []float32, k int64) (segment.VecPostingsList, error) { + search: func(qVector []float32, k int64, params json.RawMessage) (segment.VecPostingsList, error) { // 1. returned postings list (of type PostingsList) has two types of information - docNum and its score. // 2. both the values can be represented using roaring bitmaps. // 3. the Iterator (of type PostingsIterator) returned would operate in terms of VecPostings. @@ -317,7 +318,7 @@ func (sb *SegmentBase) InterpretVectorIndex(field string, except *roaring.Bitmap return rv, nil } - scores, ids, err := vecIndex.SearchWithoutIDs(qVector, k, vectorIDsToExclude) + scores, ids, err := vecIndex.SearchWithoutIDs(qVector, k, vectorIDsToExclude, params) if err != nil { return nil, err } From d792080f4c4aaa966eed29c289b4894b7a33b2a9 Mon Sep 17 00:00:00 2001 From: Abhinav Dangeti Date: Wed, 24 Jul 2024 11:23:06 -0600 Subject: [PATCH 2/3] Upgrade to bleve_index_api@v1.1.10 & scorch_segment_api@v2.2.15 --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index cee82619..8b97e9ad 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,10 @@ go 1.20 require ( github.com/RoaringBitmap/roaring v1.9.3 - github.com/blevesearch/bleve_index_api v1.1.9 + github.com/blevesearch/bleve_index_api v1.1.10 github.com/blevesearch/go-faiss v1.0.19 github.com/blevesearch/mmap-go v1.0.4 - github.com/blevesearch/scorch_segment_api/v2 v2.2.14 + github.com/blevesearch/scorch_segment_api/v2 v2.2.15 github.com/blevesearch/vellum v1.0.10 github.com/golang/snappy v0.0.1 github.com/spf13/cobra v1.7.0 diff --git a/go.sum b/go.sum index 848f09de..f04fb10a 100644 --- a/go.sum +++ b/go.sum @@ -2,14 +2,14 @@ github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4 github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA= github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= -github.com/blevesearch/bleve_index_api v1.1.9 h1:Cpq0Lp3As0Gfk3+PmcoNDRKeI50C5yuFNpj0YlN/bOE= -github.com/blevesearch/bleve_index_api v1.1.9/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8= +github.com/blevesearch/bleve_index_api v1.1.10 h1:PDLFhVjrjQWr6jCuU7TwlmByQVCSEURADHdCqVS9+g0= +github.com/blevesearch/bleve_index_api v1.1.10/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8= github.com/blevesearch/go-faiss v1.0.19 h1:UKoP8hS7DVsVSRRloNJb4qPfe2UQ99pP4D3oXd23g2A= github.com/blevesearch/go-faiss v1.0.19/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8= github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= -github.com/blevesearch/scorch_segment_api/v2 v2.2.14 h1:fgMLMpGWR7u2TdRm7XSZVWhPvMAcdYHh25Lq1fQ6Fjo= -github.com/blevesearch/scorch_segment_api/v2 v2.2.14/go.mod h1:B7+a7vfpY4NsjuTkpv/eY7RZ91Xr90VaJzT2t7upZN8= +github.com/blevesearch/scorch_segment_api/v2 v2.2.15 h1:prV17iU/o+A8FiZi9MXmqbagd8I0bCqM7OKUYPbnb5Y= +github.com/blevesearch/scorch_segment_api/v2 v2.2.15/go.mod h1:db0cmP03bPNadXrCDuVkKLV6ywFSiRgPFT1YVrestBc= github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI= github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= From 4cee9a26a8474f69f0995c992091fc650240c287 Mon Sep 17 00:00:00 2001 From: Abhinav Dangeti Date: Thu, 25 Jul 2024 09:25:03 -0600 Subject: [PATCH 3/3] Upgrade to blevesearch/go-faiss@v1.0.20 * 88dd5e2 Mohd Shaad Khan | Mb 61889: support search with params --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8b97e9ad..ae90ac25 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( github.com/RoaringBitmap/roaring v1.9.3 github.com/blevesearch/bleve_index_api v1.1.10 - github.com/blevesearch/go-faiss v1.0.19 + github.com/blevesearch/go-faiss v1.0.20 github.com/blevesearch/mmap-go v1.0.4 github.com/blevesearch/scorch_segment_api/v2 v2.2.15 github.com/blevesearch/vellum v1.0.10 diff --git a/go.sum b/go.sum index f04fb10a..1b7b237d 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZ github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/blevesearch/bleve_index_api v1.1.10 h1:PDLFhVjrjQWr6jCuU7TwlmByQVCSEURADHdCqVS9+g0= github.com/blevesearch/bleve_index_api v1.1.10/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8= -github.com/blevesearch/go-faiss v1.0.19 h1:UKoP8hS7DVsVSRRloNJb4qPfe2UQ99pP4D3oXd23g2A= -github.com/blevesearch/go-faiss v1.0.19/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8= +github.com/blevesearch/go-faiss v1.0.20 h1:AIkdTQFWuZ5LQmKQSebgMR4RynGNw8ZseJXaan5kvtI= +github.com/blevesearch/go-faiss v1.0.20/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8= github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= github.com/blevesearch/scorch_segment_api/v2 v2.2.15 h1:prV17iU/o+A8FiZi9MXmqbagd8I0bCqM7OKUYPbnb5Y=