From 80840bbd23b5574b3436e2a1bdebb6c815748fe1 Mon Sep 17 00:00:00 2001 From: CascadingRadium Date: Wed, 5 Jun 2024 18:18:57 +0530 Subject: [PATCH 1/2] MB-62167: Fix windows crash --- faiss_vector_cache.go | 3 +-- faiss_vector_io_flags_unix.go | 22 ++++++++++++++++++++++ faiss_vector_io_flags_win.go | 22 ++++++++++++++++++++++ section_faiss_vector_index.go | 3 +-- 4 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 faiss_vector_io_flags_unix.go create mode 100644 faiss_vector_io_flags_win.go diff --git a/faiss_vector_cache.go b/faiss_vector_cache.go index b9939cf8..31d1c741 100644 --- a/faiss_vector_cache.go +++ b/faiss_vector_cache.go @@ -118,8 +118,7 @@ func (vc *vectorIndexCache) createAndCache(fieldID uint16, mem []byte, except *r indexSize, n := binary.Uvarint(mem[pos : pos+binary.MaxVarintLen64]) pos += n - index, err = faiss.ReadIndexFromBuffer(mem[pos:pos+int(indexSize)], - faiss.IOFlagReadMmap|faiss.IOFlagSkipPrefetch) + index, err = faiss.ReadIndexFromBuffer(mem[pos:pos+int(indexSize)], vectorIndexIOFlags) if err != nil { return nil, nil, nil, err } diff --git a/faiss_vector_io_flags_unix.go b/faiss_vector_io_flags_unix.go new file mode 100644 index 00000000..d8fd9947 --- /dev/null +++ b/faiss_vector_io_flags_unix.go @@ -0,0 +1,22 @@ +// Copyright (c) 2024 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors && !windows +// +build vectors,!windows + +package zap + +import faiss "github.com/blevesearch/go-faiss" + +const vectorIndexIOFlags = faiss.IOFlagReadMmap | faiss.IOFlagSkipPrefetch diff --git a/faiss_vector_io_flags_win.go b/faiss_vector_io_flags_win.go new file mode 100644 index 00000000..4c9a27f0 --- /dev/null +++ b/faiss_vector_io_flags_win.go @@ -0,0 +1,22 @@ +// Copyright (c) 2024 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build vectors && windows +// +build vectors,windows + +package zap + +import faiss "github.com/blevesearch/go-faiss" + +const vectorIndexIOFlags = faiss.IOFlagReadOnly diff --git a/section_faiss_vector_index.go b/section_faiss_vector_index.go index 1eaf40a6..f01962a9 100644 --- a/section_faiss_vector_index.go +++ b/section_faiss_vector_index.go @@ -286,8 +286,7 @@ func (v *vectorIndexOpaque) mergeAndWriteVectorIndexes(sbs []*SegmentBase, } // read the index bytes. todo: parallelize this indexBytes := segBase.mem[indexes[segI].startOffset : indexes[segI].startOffset+int(indexes[segI].indexSize)] - index, err := faiss.ReadIndexFromBuffer(indexBytes, - faiss.IOFlagReadMmap|faiss.IOFlagSkipPrefetch) + index, err := faiss.ReadIndexFromBuffer(indexBytes, vectorIndexIOFlags) if err != nil { freeReconstructedIndexes(vecIndexes) return err From 3a184cd7b729e1ef1383ea372c3ba2e2152a4fab Mon Sep 17 00:00:00 2001 From: CascadingRadium Date: Thu, 6 Jun 2024 09:48:14 +0530 Subject: [PATCH 2/2] vectorIndexIOFlags -> faissIOFlags --- faiss_vector_cache.go | 2 +- faiss_vector_io_flags_unix.go | 2 +- faiss_vector_io_flags_win.go | 2 +- section_faiss_vector_index.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/faiss_vector_cache.go b/faiss_vector_cache.go index 31d1c741..893da2d5 100644 --- a/faiss_vector_cache.go +++ b/faiss_vector_cache.go @@ -118,7 +118,7 @@ func (vc *vectorIndexCache) createAndCache(fieldID uint16, mem []byte, except *r indexSize, n := binary.Uvarint(mem[pos : pos+binary.MaxVarintLen64]) pos += n - index, err = faiss.ReadIndexFromBuffer(mem[pos:pos+int(indexSize)], vectorIndexIOFlags) + index, err = faiss.ReadIndexFromBuffer(mem[pos:pos+int(indexSize)], faissIOFlags) if err != nil { return nil, nil, nil, err } diff --git a/faiss_vector_io_flags_unix.go b/faiss_vector_io_flags_unix.go index d8fd9947..a4bb8a82 100644 --- a/faiss_vector_io_flags_unix.go +++ b/faiss_vector_io_flags_unix.go @@ -19,4 +19,4 @@ package zap import faiss "github.com/blevesearch/go-faiss" -const vectorIndexIOFlags = faiss.IOFlagReadMmap | faiss.IOFlagSkipPrefetch +const faissIOFlags = faiss.IOFlagReadMmap | faiss.IOFlagSkipPrefetch diff --git a/faiss_vector_io_flags_win.go b/faiss_vector_io_flags_win.go index 4c9a27f0..de99f64d 100644 --- a/faiss_vector_io_flags_win.go +++ b/faiss_vector_io_flags_win.go @@ -19,4 +19,4 @@ package zap import faiss "github.com/blevesearch/go-faiss" -const vectorIndexIOFlags = faiss.IOFlagReadOnly +const faissIOFlags = faiss.IOFlagReadOnly diff --git a/section_faiss_vector_index.go b/section_faiss_vector_index.go index f01962a9..f4d537b3 100644 --- a/section_faiss_vector_index.go +++ b/section_faiss_vector_index.go @@ -286,7 +286,7 @@ func (v *vectorIndexOpaque) mergeAndWriteVectorIndexes(sbs []*SegmentBase, } // read the index bytes. todo: parallelize this indexBytes := segBase.mem[indexes[segI].startOffset : indexes[segI].startOffset+int(indexes[segI].indexSize)] - index, err := faiss.ReadIndexFromBuffer(indexBytes, vectorIndexIOFlags) + index, err := faiss.ReadIndexFromBuffer(indexBytes, faissIOFlags) if err != nil { freeReconstructedIndexes(vecIndexes) return err