From e8772e1d840c47a764e4ab67f5e7cf6e4d579f67 Mon Sep 17 00:00:00 2001 From: Andrew Prudhomme Date: Wed, 8 Jan 2025 12:41:15 -0800 Subject: [PATCH] Update to lucene 10.1.0 (#791) * Update to lucene 10.0.0 * Update to lucene 10.1.0 --- gradle/libs.versions.toml | 4 +- .../nrtsearch/server/codec/ServerCodec.java | 4 +- .../nrtsearch/server/doc/LoadedDocValues.java | 24 ++- .../server/facet/DrillSidewaysImpl.java | 12 +- .../server/facet/FilteredSSDVFacetCounts.java | 44 +++-- .../server/field/ContextSuggestFieldDef.java | 4 +- .../server/handler/CreateSnapshotHandler.java | 2 +- .../server/handler/RecvCopyStateHandler.java | 28 +-- .../server/handler/SearchHandler.java | 65 ++++--- .../server/handler/StatsHandler.java | 12 +- .../server/handler/WriteNRTPointHandler.java | 2 +- .../nrtsearch/server/index/ShardState.java | 20 +- .../server/index/StartIndexProcessor.java | 2 +- .../server/innerhit/InnerHitFetchTask.java | 6 +- .../server/monitoring/IndexMetrics.java | 8 +- .../nrtsearch/server/nrt/NRTReplicaNode.java | 6 +- .../nrtsearch/server/nrt/NrtDataManager.java | 18 +- .../nrtsearch/server/nrt/SimpleCopyJob.java | 4 +- .../server/nrt/state/NrtFileMetaData.java | 24 ++- .../server/nrt/state/NrtPointState.java | 13 +- .../server/query/BlendedTermQuery.java | 6 +- .../server/query/QueryNodeMapper.java | 2 +- .../nrtsearch/server/query/QueryUtils.java | 2 +- .../MultiFunctionScoreQuery.java | 79 ++++---- .../server/query/vector/ExactVectorQuery.java | 84 +++++--- .../server/rescore/QueryRescore.java | 2 +- .../server/search/MyIndexSearcher.java | 182 +++++------------- .../server/search/SearchRequestProcessor.java | 10 +- .../additional/FilterCollectorManager.java | 4 +- .../OrdinalTermsCollectorManager.java | 2 +- .../additional/TopHitsCollectorManager.java | 6 +- .../lucene/replicator/nrt/CopyOneFile.java | 8 +- .../server/codec/ServerCodecTest.java | 6 +- .../server/doc/SharedDocContextTest.java | 2 +- .../server/facet/EagerGlobalOrdinalsTest.java | 14 +- .../server/field/EagerFieldOrdinalTest.java | 14 +- .../server/field/VectorFieldDefTest.java | 17 +- .../server/grpc/MultiSegmentTest.java | 4 +- .../nrtsearch/server/grpc/TimeoutTest.java | 14 +- .../index/BucketedTieredMergePolicyTest.java | 37 ++-- .../server/index/DirectoryFactoryTest.java | 77 -------- .../nrt/PrimaryNodeReferenceManagerTest.java | 13 +- .../server/nrt/state/NrtPointStateTest.java | 20 +- .../server/rescore/RescorerTest.java | 2 +- .../search/GlobalOrdinalLookupTest.java | 14 +- .../server/search/MyIndexSearcherTest.java | 38 ++-- .../MyIndexSearcherVirtualShardsTest.java | 21 +- .../MyTopSuggestDocsCollectorTest.java | 8 +- .../server/state/BackendGlobalStateTest.java | 7 +- 49 files changed, 442 insertions(+), 555 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a734216b9..37aa23dc5 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -3,7 +3,7 @@ aws = "1.12.768" grpc = "1.66.0" jackson = "2.17.2" log4j = "2.23.1" -lucene = "9.12.0" +lucene = "10.1.0" prometheus = "1.3.1" protobuf = "3.25.3" @@ -59,7 +59,7 @@ grpc-inprocess = { module = "io.grpc:grpc-inprocess", version.ref = "grpc" } grpc-testing = { module = "io.grpc:grpc-testing", version.ref = "grpc" } junit = { module = "junit:junit", version = "4.13.2" } lucene-test-framework = { module = "org.apache.lucene:lucene-test-framework", version.ref = "lucene" } -mockito-core = { module = "org.mockito:mockito-core", version = "5.12.0" } +mockito-core = { module = "org.mockito:mockito-core", version = "5.14.2" } s3mock = { module = "io.findify:s3mock_2.13", version = "0.2.6" } spatial4j = { module = "org.locationtech.spatial4j:spatial4j", version = "0.8" } diff --git a/src/main/java/com/yelp/nrtsearch/server/codec/ServerCodec.java b/src/main/java/com/yelp/nrtsearch/server/codec/ServerCodec.java index cab2877a0..16f18f0c6 100644 --- a/src/main/java/com/yelp/nrtsearch/server/codec/ServerCodec.java +++ b/src/main/java/com/yelp/nrtsearch/server/codec/ServerCodec.java @@ -24,10 +24,10 @@ import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene912.Lucene912Codec; +import org.apache.lucene.codecs.lucene101.Lucene101Codec; /** Implements per-index {@link Codec}. */ -public class ServerCodec extends Lucene912Codec { +public class ServerCodec extends Lucene101Codec { private final IndexStateManager stateManager; // nocommit expose compression control diff --git a/src/main/java/com/yelp/nrtsearch/server/doc/LoadedDocValues.java b/src/main/java/com/yelp/nrtsearch/server/doc/LoadedDocValues.java index 976a27a3c..fb8b94b2d 100644 --- a/src/main/java/com/yelp/nrtsearch/server/doc/LoadedDocValues.java +++ b/src/main/java/com/yelp/nrtsearch/server/doc/LoadedDocValues.java @@ -1075,20 +1075,22 @@ public ByteVectorType getValue() { */ public static final class SingleSearchVector extends LoadedDocValues { private final FloatVectorValues vectorValues; + private final KnnVectorValues.DocIndexIterator vectorIterator; private FloatVectorType value = null; public SingleSearchVector(FloatVectorValues vectorValues) { this.vectorValues = vectorValues; + this.vectorIterator = vectorValues != null ? vectorValues.iterator() : null; } @Override public void setDocId(int docID) throws IOException { - if (vectorValues != null) { - if (vectorValues.docID() < docID) { - vectorValues.advance(docID); + if (vectorIterator != null) { + if (vectorIterator.docID() < docID) { + vectorIterator.advance(docID); } - if (vectorValues.docID() == docID) { - value = new FloatVectorType(vectorValues.vectorValue()); + if (vectorIterator.docID() == docID) { + value = new FloatVectorType(vectorValues.vectorValue(vectorIterator.index())); } else { value = null; } @@ -1133,20 +1135,22 @@ public FloatVectorType getValue() { */ public static final class SingleSearchByteVector extends LoadedDocValues { private final ByteVectorValues vectorValues; + private final KnnVectorValues.DocIndexIterator vectorIterator; private ByteVectorType value = null; public SingleSearchByteVector(ByteVectorValues vectorValues) { this.vectorValues = vectorValues; + this.vectorIterator = vectorValues != null ? vectorValues.iterator() : null; } @Override public void setDocId(int docID) throws IOException { - if (vectorValues != null) { - if (vectorValues.docID() < docID) { - vectorValues.advance(docID); + if (vectorIterator != null) { + if (vectorIterator.docID() < docID) { + vectorIterator.advance(docID); } - if (vectorValues.docID() == docID) { - value = new ByteVectorType(vectorValues.vectorValue()); + if (vectorIterator.docID() == docID) { + value = new ByteVectorType(vectorValues.vectorValue(vectorIterator.index())); } else { value = null; } diff --git a/src/main/java/com/yelp/nrtsearch/server/facet/DrillSidewaysImpl.java b/src/main/java/com/yelp/nrtsearch/server/facet/DrillSidewaysImpl.java index 867c52289..447d4d7ee 100644 --- a/src/main/java/com/yelp/nrtsearch/server/facet/DrillSidewaysImpl.java +++ b/src/main/java/com/yelp/nrtsearch/server/facet/DrillSidewaysImpl.java @@ -191,8 +191,8 @@ private static com.yelp.nrtsearch.server.grpc.FacetResult getScriptFacetResult( int totalDocs = 0; // run script against all match docs, and aggregate counts for (MatchingDocs matchingDocs : drillDowns.getMatchingDocs()) { - FacetScript script = segmentFactory.newInstance(matchingDocs.context); - DocIdSetIterator iterator = matchingDocs.bits.iterator(); + FacetScript script = segmentFactory.newInstance(matchingDocs.context()); + DocIdSetIterator iterator = matchingDocs.bits().iterator(); if (iterator == null) { continue; } @@ -216,8 +216,8 @@ private static com.yelp.nrtsearch.server.grpc.FacetResult getDocValuesFacetResul int totalDocs = 0; // get doc values for all match docs, and aggregate counts for (MatchingDocs matchingDocs : drillDowns.getMatchingDocs()) { - LoadedDocValues docValues = fieldDef.getDocValues(matchingDocs.context); - DocIdSetIterator iterator = matchingDocs.bits.iterator(); + LoadedDocValues docValues = fieldDef.getDocValues(matchingDocs.context()); + DocIdSetIterator iterator = matchingDocs.bits().iterator(); if (iterator == null) { continue; } @@ -462,7 +462,7 @@ private static com.yelp.nrtsearch.server.grpc.FacetResult getFieldFacetResult( luceneFacets = new FastTaxonomyFacetCounts( indexFieldName, - searcherAndTaxonomyManager.taxonomyReader, + searcherAndTaxonomyManager.taxonomyReader(), indexState.getFacetsConfig(), c); } else { @@ -478,7 +478,7 @@ private static com.yelp.nrtsearch.server.grpc.FacetResult getFieldFacetResult( luceneFacets = new FastTaxonomyFacetCounts( indexFieldName, - searcherAndTaxonomyManager.taxonomyReader, + searcherAndTaxonomyManager.taxonomyReader(), indexState.getFacetsConfig(), drillDowns); indexFieldNameToFacets.put(indexFieldName, luceneFacets); diff --git a/src/main/java/com/yelp/nrtsearch/server/facet/FilteredSSDVFacetCounts.java b/src/main/java/com/yelp/nrtsearch/server/facet/FilteredSSDVFacetCounts.java index 954af5a74..6d6c81859 100644 --- a/src/main/java/com/yelp/nrtsearch/server/facet/FilteredSSDVFacetCounts.java +++ b/src/main/java/com/yelp/nrtsearch/server/facet/FilteredSSDVFacetCounts.java @@ -58,7 +58,7 @@ public class FilteredSSDVFacetCounts extends Facets { final String field; final Map globalOrdinalToCountIndex; final List values; - final int[] counts; + int[] counts; /** * Facet to count based on sorted set doc values, but only considering the provided values. @@ -76,7 +76,6 @@ public FilteredSSDVFacetCounts( this.field = state.getField(); this.values = values; dv = state.getDocValues(); - counts = new int[values.size()]; // find mapping to go from global ordinal to the value count index globalOrdinalToCountIndex = new HashMap<>(); @@ -98,6 +97,12 @@ public FilteredSSDVFacetCounts( } } + private void initializeCounts() { + if (counts == null) { + counts = new int[values.size()]; + } + } + /** Does all the "real work" of tallying up the counts. */ private void count(List matchingDocs) throws IOException { OrdinalMap ordinalMap; @@ -117,12 +122,12 @@ private void count(List matchingDocs) throws IOException { // the top-level reader passed to the // SortedSetDocValuesReaderState, else cryptic // AIOOBE can happen: - if (ReaderUtil.getTopLevelContext(hits.context).reader() != reader) { + if (ReaderUtil.getTopLevelContext(hits.context()).reader() != reader) { throw new IllegalStateException( "the SortedSetDocValuesReaderState provided to this class does not match the reader being searched; you must create a new SortedSetDocValuesReaderState every time you open a new IndexReader"); } - countOneSegment(ordinalMap, hits.context.reader(), hits.context.ord, hits, null); + countOneSegment(ordinalMap, hits.context().reader(), hits.context().ord, hits, null); } } @@ -135,6 +140,9 @@ private void countOneSegmentNHLD(OrdinalMap ordinalMap, LeafReader reader, int s return; } + // Initialize counts: + initializeCounts(); + // It's slightly more efficient to work against SortedDocValues if the field is actually // single-valued (see: LUCENE-5309) SortedDocValues singleValues = DocValues.unwrapSingleton(multiValues); @@ -177,7 +185,7 @@ private void countOneSegmentNHLD(OrdinalMap ordinalMap, LeafReader reader, int s if (count != 0) { Integer countIndex = globalOrdinalToCountIndex.get(ordMap.get(ord)); if (countIndex != null) { - counts[countIndex]++; + counts[countIndex] += count; } } } @@ -212,12 +220,19 @@ private void countOneSegmentNHLD(OrdinalMap ordinalMap, LeafReader reader, int s private void countOneSegment( OrdinalMap ordinalMap, LeafReader reader, int segOrd, MatchingDocs hits, Bits liveDocs) throws IOException { + if (hits != null && hits.totalHits() == 0) { + return; + } + SortedSetDocValues multiValues = DocValues.getSortedSet(reader, field); if (multiValues == null) { // nothing to count return; } + // Initialize counts: + initializeCounts(); + // It's slightly more efficient to work against SortedDocValues if the field is actually // single-valued (see: LUCENE-5309) SortedDocValues singleValues = DocValues.unwrapSingleton(multiValues); @@ -229,7 +244,7 @@ private void countOneSegment( it = FacetUtils.liveDocsDISI(valuesIt, liveDocs); ; } else { - it = ConjunctionUtils.intersectIterators(Arrays.asList(hits.bits.iterator(), valuesIt)); + it = ConjunctionUtils.intersectIterators(Arrays.asList(hits.bits().iterator(), valuesIt)); } // TODO: yet another option is to count all segs @@ -246,7 +261,7 @@ private void countOneSegment( int numSegOrds = (int) multiValues.getValueCount(); - if (hits != null && hits.totalHits < numSegOrds / 10) { + if (hits != null && hits.totalHits() < numSegOrds / 10) { // Remap every ord to global ord as we iterate: if (singleValues != null) { for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) { @@ -288,7 +303,7 @@ private void countOneSegment( if (count != 0) { Integer countIndex = globalOrdinalToCountIndex.get(ordMap.get(ord)); if (countIndex != null) { - counts[countIndex]++; + counts[countIndex] += count; } } } @@ -337,8 +352,7 @@ private void countAll() throws IOException { if (liveDocs == null) { countOneSegmentNHLD(ordinalMap, context.reader(), context.ord); } else { - countOneSegment( - ordinalMap, context.reader(), context.ord, null, context.reader().getLiveDocs()); + countOneSegment(ordinalMap, context.reader(), context.ord, null, liveDocs); } } } @@ -366,14 +380,14 @@ private FacetResult getDim(String dim, int topN) throws IOException { int dimCount = 0; int childCount = 0; - TopOrdAndIntQueue.OrdAndValue reuse = null; + TopOrdAndIntQueue.OrdAndInt reuse = null; for (int ord = 0; ord < counts.length; ord++) { if (counts[ord] > 0) { dimCount += counts[ord]; childCount++; if (counts[ord] > bottomCount) { if (reuse == null) { - reuse = new TopOrdAndIntQueue.OrdAndValue(); + reuse = new TopOrdAndIntQueue.OrdAndInt(); } reuse.ord = ord; reuse.value = counts[ord]; @@ -382,9 +396,9 @@ private FacetResult getDim(String dim, int topN) throws IOException { // sparse case unnecessarily q = new TopOrdAndIntQueue(topN); } - reuse = q.insertWithOverflow(reuse); + reuse = (TopOrdAndIntQueue.OrdAndInt) q.insertWithOverflow(reuse); if (q.size() == topN) { - bottomCount = q.top().value; + bottomCount = ((TopOrdAndIntQueue.OrdAndInt) q.top()).value; } } } @@ -396,7 +410,7 @@ private FacetResult getDim(String dim, int topN) throws IOException { LabelAndValue[] labelValues = new LabelAndValue[q.size()]; for (int i = labelValues.length - 1; i >= 0; i--) { - TopOrdAndIntQueue.OrdAndValue ordAndValue = q.pop(); + TopOrdAndIntQueue.OrdAndInt ordAndValue = (TopOrdAndIntQueue.OrdAndInt) q.pop(); labelValues[i] = new LabelAndValue(values.get(ordAndValue.ord), ordAndValue.value); } return new FacetResult(dim, new String[0], dimCount, labelValues, childCount); diff --git a/src/main/java/com/yelp/nrtsearch/server/field/ContextSuggestFieldDef.java b/src/main/java/com/yelp/nrtsearch/server/field/ContextSuggestFieldDef.java index 3a4a99a79..74c32c8a0 100644 --- a/src/main/java/com/yelp/nrtsearch/server/field/ContextSuggestFieldDef.java +++ b/src/main/java/com/yelp/nrtsearch/server/field/ContextSuggestFieldDef.java @@ -25,7 +25,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.document.Document; -import org.apache.lucene.search.suggest.document.Completion912PostingsFormat; +import org.apache.lucene.search.suggest.document.Completion101PostingsFormat; import org.apache.lucene.search.suggest.document.ContextSuggestField; public class ContextSuggestFieldDef extends IndexableFieldDef { @@ -45,7 +45,7 @@ protected ContextSuggestFieldDef( this.indexAnalyzer = this.parseIndexAnalyzer(requestField); this.searchAnalyzer = this.parseSearchAnalyzer(requestField); this.postingsFormat = - new Completion912PostingsFormat(context.config().getCompletionCodecLoadMode()); + new Completion101PostingsFormat(context.config().getCompletionCodecLoadMode()); } @Override diff --git a/src/main/java/com/yelp/nrtsearch/server/handler/CreateSnapshotHandler.java b/src/main/java/com/yelp/nrtsearch/server/handler/CreateSnapshotHandler.java index bcbc8cb91..cec62161c 100644 --- a/src/main/java/com/yelp/nrtsearch/server/handler/CreateSnapshotHandler.java +++ b/src/main/java/com/yelp/nrtsearch/server/handler/CreateSnapshotHandler.java @@ -86,7 +86,7 @@ private CreateSnapshotResponse createSnapshot( // search is done: long t0 = System.nanoTime(); IndexReader r = - DirectoryReader.openIfChanged((DirectoryReader) s2.searcher.getIndexReader(), c); + DirectoryReader.openIfChanged((DirectoryReader) s2.searcher().getIndexReader(), c); IndexSearcher s = new IndexSearcher(r); try { shardState.slm.record(s); diff --git a/src/main/java/com/yelp/nrtsearch/server/handler/RecvCopyStateHandler.java b/src/main/java/com/yelp/nrtsearch/server/handler/RecvCopyStateHandler.java index ba6c00ba9..a5eb38066 100644 --- a/src/main/java/com/yelp/nrtsearch/server/handler/RecvCopyStateHandler.java +++ b/src/main/java/com/yelp/nrtsearch/server/handler/RecvCopyStateHandler.java @@ -88,22 +88,22 @@ private CopyState handle(IndexState indexState, CopyStateRequest copyStateReques private static CopyState writeCopyState(org.apache.lucene.replicator.nrt.CopyState state) throws IOException { CopyState.Builder builder = CopyState.newBuilder(); - builder.setInfoBytesLength(state.infosBytes.length); - builder.setInfoBytes(ByteString.copyFrom(state.infosBytes, 0, state.infosBytes.length)); + builder.setInfoBytesLength(state.infosBytes().length); + builder.setInfoBytes(ByteString.copyFrom(state.infosBytes(), 0, state.infosBytes().length)); - builder.setGen(state.gen); - builder.setVersion(state.version); + builder.setGen(state.gen()); + builder.setVersion(state.version()); - FilesMetadata filesMetadata = writeFilesMetaData(state.files); + FilesMetadata filesMetadata = writeFilesMetaData(state.files()); builder.setFilesMetadata(filesMetadata); - builder.setCompletedMergeFilesSize(state.completedMergeFiles.size()); + builder.setCompletedMergeFilesSize(state.completedMergeFiles().size()); - for (String fileName : state.completedMergeFiles) { + for (String fileName : state.completedMergeFiles()) { builder.addCompletedMergeFiles(fileName); } - builder.setPrimaryGen(state.primaryGen); + builder.setPrimaryGen(state.primaryGen()); return builder.build(); } @@ -117,12 +117,12 @@ public static FilesMetadata writeFilesMetaData(Map files) fileMetadataBuilder.setFileName(ent.getKey()); FileMetaData fmd = ent.getValue(); - fileMetadataBuilder.setLen(fmd.length); - fileMetadataBuilder.setChecksum(fmd.checksum); - fileMetadataBuilder.setHeaderLength(fmd.header.length); - fileMetadataBuilder.setHeader(ByteString.copyFrom(fmd.header, 0, fmd.header.length)); - fileMetadataBuilder.setFooterLength(fmd.footer.length); - fileMetadataBuilder.setFooter(ByteString.copyFrom(fmd.footer, 0, fmd.footer.length)); + fileMetadataBuilder.setLen(fmd.length()); + fileMetadataBuilder.setChecksum(fmd.checksum()); + fileMetadataBuilder.setHeaderLength(fmd.header().length); + fileMetadataBuilder.setHeader(ByteString.copyFrom(fmd.header(), 0, fmd.header().length)); + fileMetadataBuilder.setFooterLength(fmd.footer().length); + fileMetadataBuilder.setFooter(ByteString.copyFrom(fmd.footer(), 0, fmd.footer().length)); builder.addFileMetadata(fileMetadataBuilder.build()); } return builder.build(); diff --git a/src/main/java/com/yelp/nrtsearch/server/handler/SearchHandler.java b/src/main/java/com/yelp/nrtsearch/server/handler/SearchHandler.java index 6f9681b53..9b966a523 100644 --- a/src/main/java/com/yelp/nrtsearch/server/handler/SearchHandler.java +++ b/src/main/java/com/yelp/nrtsearch/server/handler/SearchHandler.java @@ -185,9 +185,9 @@ public SearchResponse handle(IndexState indexState, SearchRequest searchRequest) // sideways task per query. DrillSideways drillS = new DrillSidewaysImpl( - s.searcher, + s.searcher(), indexState.getFacetsConfig(), - s.taxonomyReader, + s.taxonomyReader(), searchRequest.getFacetsList(), s, indexState, @@ -219,13 +219,14 @@ public SearchResponse handle(IndexState indexState, SearchRequest searchRequest) searcherResult.getTopDocs(), searchRequest.getFacetsList(), indexState, - s.searcher, + s.searcher(), diagnostics)); } else { try { searcherResult = - s.searcher.search( - searchContext.getQuery(), searchContext.getCollector().getWrappedManager()); + s.searcher() + .search( + searchContext.getQuery(), searchContext.getCollector().getWrappedManager()); } catch (RuntimeException e) { CollectionTimeoutException timeoutException = findTimeoutException(e); if (timeoutException != null) { @@ -296,7 +297,8 @@ public SearchResponse handle(IndexState indexState, SearchRequest searchRequest) searchState.setTimestamp(searchContext.getTimestampSec()); // Record searcher version that handled this request: - searchState.setSearcherVersion(((DirectoryReader) s.searcher.getIndexReader()).getVersion()); + searchState.setSearcherVersion( + ((DirectoryReader) s.searcher().getIndexReader()).getVersion()); // Fill in lastDoc for searchAfter: if (hits.scoreDocs.length != 0) { @@ -403,7 +405,7 @@ private void fetchFields(SearchContext searchContext) // Fetch fields in parallel List leaves = - searchContext.getSearcherAndTaxonomy().searcher.getIndexReader().leaves(); + searchContext.getSearcherAndTaxonomy().searcher().getIndexReader().leaves(); List hitIdToLeaves = new ArrayList<>(); for (int hitIndex = 0; hitIndex < hitBuilders.size(); ++hitIndex) { var hitResponse = hitBuilders.get(hitIndex); @@ -433,7 +435,7 @@ private void fetchFields(SearchContext searchContext) .fetchExecutor() .submit( new FillFieldsTask( - searchContext.getSearcherAndTaxonomy().searcher, + searchContext.getSearcherAndTaxonomy().searcher(), hitIdToLeaves, hitBuilders, fieldsChunk, @@ -455,7 +457,7 @@ private void fetchFields(SearchContext searchContext) hitResponse.setExplain( searchContext .getSearcherAndTaxonomy() - .searcher + .searcher() .explain(searchContext.getQuery(), hitResponse.getLuceneDocId()) .toString()); } @@ -551,8 +553,8 @@ private static void setResponseTopHits(SearchContext context) { private static void setResponseHits(SearchContext context, TopDocs hits) { TotalHits totalHits = TotalHits.newBuilder() - .setRelation(TotalHits.Relation.valueOf(hits.totalHits.relation.name())) - .setValue(hits.totalHits.value) + .setRelation(TotalHits.Relation.valueOf(hits.totalHits.relation().name())) + .setValue(hits.totalHits.value()) .build(); context.getResponseBuilder().setTotalHits(totalHits); for (int hitIndex = 0; hitIndex < hits.scoreDocs.length; hitIndex++) { @@ -604,7 +606,8 @@ public static SearcherTaxonomyManager.SearcherAndTaxonomy getSearcherAndTaxonomy s = openSnapshotReader(indexState, state, snapshot, diagnostics, searchExecutor); } else { SearcherTaxonomyManager.SearcherAndTaxonomy current = state.acquire(); - long currentVersion = ((DirectoryReader) current.searcher.getIndexReader()).getVersion(); + long currentVersion = + ((DirectoryReader) current.searcher().getIndexReader()).getVersion(); if (currentVersion == version) { s = current; } else if (version > currentVersion) { @@ -639,9 +642,9 @@ public void afterRefresh(boolean didRefresh) throws IOException { SearcherTaxonomyManager.SearcherAndTaxonomy current = state.acquire(); logger.info( "SearchHandler: refresh completed newVersion=" - + ((DirectoryReader) current.searcher.getIndexReader()).getVersion()); + + ((DirectoryReader) current.searcher().getIndexReader()).getVersion()); try { - if (((DirectoryReader) current.searcher.getIndexReader()).getVersion() + if (((DirectoryReader) current.searcher().getIndexReader()).getVersion() >= version) { lock.lock(); try { @@ -660,17 +663,17 @@ public void afterRefresh(boolean didRefresh) throws IOException { lock.lock(); try { current = state.acquire(); - if (((DirectoryReader) current.searcher.getIndexReader()).getVersion() < version) { + if (((DirectoryReader) current.searcher().getIndexReader()).getVersion() < version) { // still not there yet state.release(current); cond.await(); current = state.acquire(); logger.info( "SearchHandler: await released, current version " - + ((DirectoryReader) current.searcher.getIndexReader()).getVersion() + + ((DirectoryReader) current.searcher().getIndexReader()).getVersion() + " required minimum version " + version); - assert ((DirectoryReader) current.searcher.getIndexReader()).getVersion() + assert ((DirectoryReader) current.searcher().getIndexReader()).getVersion() >= version; } s = current; @@ -695,8 +698,8 @@ public void afterRefresh(boolean didRefresh) throws IOException { // but the latest taxoReader ... necessary // because SLM can't take taxo reader yet: SearcherTaxonomyManager.SearcherAndTaxonomy s2 = state.acquire(); - s = new SearcherTaxonomyManager.SearcherAndTaxonomy(priorSearcher, s2.taxonomyReader); - s2.searcher.getIndexReader().decRef(); + s = new SearcherTaxonomyManager.SearcherAndTaxonomy(priorSearcher, s2.taxonomyReader()); + s2.searcher().getIndexReader().decRef(); } } else if (searchCase.equals((SearchRequest.SearcherCase.INDEXGEN))) { // Searcher is identified by an indexGen, returned @@ -718,12 +721,12 @@ public void afterRefresh(boolean didRefresh) throws IOException { diagnostics.setNrtWaitTimeMs((System.nanoTime() - t0) / 1000000.0); } s = state.acquire(); - state.slm.record(s.searcher); + state.slm.record(s.searcher()); } else if (searchCase.equals(SearchRequest.SearcherCase.SEARCHER_NOT_SET)) { // Request didn't specify any specific searcher; // just use the current (latest) searcher: s = state.acquire(); - state.slm.record(s.searcher); + state.slm.record(s.searcher()); } else { throw new UnsupportedOperationException(searchCase.name() + " is not yet supported "); } @@ -755,23 +758,23 @@ private static SearcherTaxonomyManager.SearcherAndTaxonomy openSnapshotReader( // Returns a ref, which we return to caller: IndexReader r = DirectoryReader.openIfChanged( - (DirectoryReader) s.searcher.getIndexReader(), + (DirectoryReader) s.searcher().getIndexReader(), state.snapshots.getIndexCommit(snapshot.indexGen)); // Ref that we return to caller - s.taxonomyReader.incRef(); + s.taxonomyReader().incRef(); SearcherTaxonomyManager.SearcherAndTaxonomy result = new SearcherTaxonomyManager.SearcherAndTaxonomy( - new MyIndexSearcher( + MyIndexSearcher.create( r, - new MyIndexSearcher.ExecutorWithParams( - searchExecutor, + searchExecutor, + new MyIndexSearcher.SlicingParams( indexState.getSliceMaxDocs(), indexState.getSliceMaxSegments(), indexState.getVirtualShards())), - s.taxonomyReader); - state.slm.record(result.searcher); + s.taxonomyReader()); + state.slm.record(result.searcher()); long t1 = System.nanoTime(); if (diagnostics != null) { diagnostics.setNewSnapshotSearcherOpenMs(((t1 - t0) / 1000000.0)); @@ -940,7 +943,7 @@ static StoredFieldFetchContext getStoredFieldFetchContext(FieldFetchContext fiel } } return new StoredFieldFetchContext( - fieldFetchContext.getSearcherAndTaxonomy().searcher.storedFields(), + fieldFetchContext.getSearcherAndTaxonomy().searcher().storedFields(), storedFieldNames, storedFieldEntries); } @@ -965,7 +968,7 @@ public FillDocsTask( @Override public void run() { List leaves = - fieldFetchContext.getSearcherAndTaxonomy().searcher.getIndexReader().leaves(); + fieldFetchContext.getSearcherAndTaxonomy().searcher().getIndexReader().leaves(); StoredFieldFetchContext storedFieldFetchContext; try { storedFieldFetchContext = getStoredFieldFetchContext(fieldFetchContext); @@ -1067,7 +1070,7 @@ private static void fetchSlice( hit.setExplain( context .getSearcherAndTaxonomy() - .searcher + .searcher() .explain(resolvedExplainQuery, hit.getLuceneDocId()) .toString()); } diff --git a/src/main/java/com/yelp/nrtsearch/server/handler/StatsHandler.java b/src/main/java/com/yelp/nrtsearch/server/handler/StatsHandler.java index 0c9b8f6f1..ab0c9a752 100644 --- a/src/main/java/com/yelp/nrtsearch/server/handler/StatsHandler.java +++ b/src/main/java/com/yelp/nrtsearch/server/handler/StatsHandler.java @@ -108,15 +108,15 @@ public boolean doPrune(double ageSec, IndexSearcher indexSearcher) { SearcherTaxonomyManager.SearcherAndTaxonomy s = shardState.acquire(); try { Taxonomy.Builder taxonomy = Taxonomy.newBuilder(); - if (s.taxonomyReader != null) { // taxo reader is null for primary and replica - taxonomy.setNumOrds(s.taxonomyReader.getSize()); - taxonomy.setSegments(s.taxonomyReader.toString()); + if (s.taxonomyReader() != null) { // taxo reader is null for primary and replica + taxonomy.setNumOrds(s.taxonomyReader().getSize()); + taxonomy.setSegments(s.taxonomyReader().toString()); } statsResponseBuilder.setTaxonomy(taxonomy.build()); Searcher.Builder searcher = Searcher.newBuilder(); - if (s.searcher != null) { - searcher.setSegments(s.searcher.toString()); - IndexReader indexReader = s.searcher.getIndexReader(); + if (s.searcher() != null) { + searcher.setSegments(s.searcher().toString()); + IndexReader indexReader = s.searcher().getIndexReader(); searcher.setNumDocs(indexReader.numDocs()); if (indexReader instanceof StandardDirectoryReader standardDirectoryReader) { searcher.setNumSegments(standardDirectoryReader.getSegmentInfos().asList().size()); diff --git a/src/main/java/com/yelp/nrtsearch/server/handler/WriteNRTPointHandler.java b/src/main/java/com/yelp/nrtsearch/server/handler/WriteNRTPointHandler.java index 18ee38f3c..5a97b0008 100644 --- a/src/main/java/com/yelp/nrtsearch/server/handler/WriteNRTPointHandler.java +++ b/src/main/java/com/yelp/nrtsearch/server/handler/WriteNRTPointHandler.java @@ -114,7 +114,7 @@ private SearcherVersion handle(IndexState indexState, String indexId) { } else { SearcherTaxonomyManager.SearcherAndTaxonomy s = shardState.acquire(); try { - long version = ((DirectoryReader) s.searcher.getIndexReader()).getVersion(); + long version = ((DirectoryReader) s.searcher().getIndexReader()).getVersion(); return searchverVersionBuilder.setVersion(version).setDidRefresh(false).build(); } finally { shardState.release(s); diff --git a/src/main/java/com/yelp/nrtsearch/server/index/ShardState.java b/src/main/java/com/yelp/nrtsearch/server/index/ShardState.java index 1869521b6..cfa12cf62 100644 --- a/src/main/java/com/yelp/nrtsearch/server/index/ShardState.java +++ b/src/main/java/com/yelp/nrtsearch/server/index/ShardState.java @@ -414,9 +414,9 @@ public SearcherTaxonomyManager.SearcherAndTaxonomy acquire() throws IOException public void release(SearcherTaxonomyManager.SearcherAndTaxonomy s) throws IOException { if (nrtPrimaryNode != null) { - nrtPrimaryNode.getSearcherManager().release(s.searcher); + nrtPrimaryNode.getSearcherManager().release(s.searcher()); } else if (nrtReplicaNode != null) { - nrtReplicaNode.getSearcherManager().release(s.searcher); + nrtReplicaNode.getSearcherManager().release(s.searcher()); } else { manager.release(s); } @@ -504,10 +504,10 @@ public IndexSearcher newSearcher(IndexReader reader, IndexReader previousReader) throws IOException { IndexState indexState = indexStateManager.getCurrent(); IndexSearcher searcher = - new MyIndexSearcher( + MyIndexSearcher.create( reader, - new MyIndexSearcher.ExecutorWithParams( - searchExecutor, + searchExecutor, + new MyIndexSearcher.SlicingParams( indexState.getSliceMaxDocs(), indexState.getSliceMaxSegments(), indexState.getVirtualShards())); @@ -788,10 +788,10 @@ public synchronized void startPrimary(NrtDataManager nrtDataManager, long primar @Override public IndexSearcher newSearcher(IndexReader r, IndexReader previousReader) { IndexSearcher searcher = - new MyIndexSearcher( + MyIndexSearcher.create( r, - new MyIndexSearcher.ExecutorWithParams( - searchExecutor, + searchExecutor, + new MyIndexSearcher.SlicingParams( indexState.getSliceMaxDocs(), indexState.getSliceMaxSegments(), indexState.getVirtualShards())); @@ -830,7 +830,7 @@ public IndexSearcher newSearcher(IndexReader r, IndexReader previousReader) { public SortedSetDocValuesReaderState getSSDVState( IndexState indexState, SearcherTaxonomyManager.SearcherAndTaxonomy s, FieldDef fd) throws IOException { - return getSSDVStateForReader(indexState, s.searcher.getIndexReader(), fd); + return getSSDVStateForReader(indexState, s.searcher().getIndexReader(), fd); } public SortedSetDocValuesReaderState getSSDVStateForReader( @@ -988,7 +988,7 @@ public void beforeRefresh() {} public void afterRefresh(boolean didRefresh) throws IOException { SearcherTaxonomyManager.SearcherAndTaxonomy current = acquire(); try { - slm.record(current.searcher); + slm.record(current.searcher()); } finally { release(current); } diff --git a/src/main/java/com/yelp/nrtsearch/server/index/StartIndexProcessor.java b/src/main/java/com/yelp/nrtsearch/server/index/StartIndexProcessor.java index b8ef3c657..44ca3e87e 100644 --- a/src/main/java/com/yelp/nrtsearch/server/index/StartIndexProcessor.java +++ b/src/main/java/com/yelp/nrtsearch/server/index/StartIndexProcessor.java @@ -144,7 +144,7 @@ private StartIndexResponse processInternal( throw new StartIndexProcessorException(e); } try { - IndexReader r = s.searcher.getIndexReader(); + IndexReader r = s.searcher().getIndexReader(); startIndexResponseBuilder.setMaxDoc(r.maxDoc()); startIndexResponseBuilder.setNumDocs(r.numDocs()); startIndexResponseBuilder.setSegments(r.toString()); diff --git a/src/main/java/com/yelp/nrtsearch/server/innerhit/InnerHitFetchTask.java b/src/main/java/com/yelp/nrtsearch/server/innerhit/InnerHitFetchTask.java index 782a1e8bd..694cc5ad1 100644 --- a/src/main/java/com/yelp/nrtsearch/server/innerhit/InnerHitFetchTask.java +++ b/src/main/java/com/yelp/nrtsearch/server/innerhit/InnerHitFetchTask.java @@ -70,7 +70,7 @@ public InnerHitContext getInnerHitContext() { public InnerHitFetchTask(InnerHitContext innerHitContext) throws IOException { this.innerHitContext = innerHitContext; - this.searcher = innerHitContext.getSearcherAndTaxonomy().searcher; + this.searcher = innerHitContext.getSearcherAndTaxonomy().searcher(); boolean needScore = innerHitContext.getTopHits() >= innerHitContext.getStartHit() && (innerHitContext.getSortContext() == null @@ -124,8 +124,8 @@ public void processHit( HitsResult.Builder innerHitResultBuilder = HitsResult.newBuilder(); TotalHits totalInnerHits = TotalHits.newBuilder() - .setRelation(TotalHits.Relation.valueOf(topDocs.totalHits.relation.name())) - .setValue(topDocs.totalHits.value) + .setRelation(TotalHits.Relation.valueOf(topDocs.totalHits.relation().name())) + .setValue(topDocs.totalHits.value()) .build(); innerHitResultBuilder.setTotalHits(totalInnerHits); diff --git a/src/main/java/com/yelp/nrtsearch/server/monitoring/IndexMetrics.java b/src/main/java/com/yelp/nrtsearch/server/monitoring/IndexMetrics.java index b149c587f..7b85d2ffd 100644 --- a/src/main/java/com/yelp/nrtsearch/server/monitoring/IndexMetrics.java +++ b/src/main/java/com/yelp/nrtsearch/server/monitoring/IndexMetrics.java @@ -124,7 +124,7 @@ public static void updateSearcherStats(String index, IndexSearcher searcher) { // segments per slice int[] segments = new int[slices.length]; for (int i = 0; i < slices.length; ++i) { - segments[i] = slices[i].leaves.length; + segments[i] = slices[i].partitions.length; } Arrays.sort(segments); sliceSegments.labelValues(index, "min").set(segments[0]); @@ -136,11 +136,7 @@ public static void updateSearcherStats(String index, IndexSearcher searcher) { // docs per slice long[] docCounts = new long[slices.length]; for (int i = 0; i < slices.length; ++i) { - long sliceCount = 0; - for (int j = 0; j < slices[i].leaves.length; ++j) { - sliceCount += slices[i].leaves[j].reader().maxDoc(); - } - docCounts[i] = sliceCount; + docCounts[i] = slices[i].getMaxDocs(); } Arrays.sort(docCounts); sliceDocs.labelValues(index, "min").set(docCounts[0]); diff --git a/src/main/java/com/yelp/nrtsearch/server/nrt/NRTReplicaNode.java b/src/main/java/com/yelp/nrtsearch/server/nrt/NRTReplicaNode.java index 88b6f299f..120c5af9c 100644 --- a/src/main/java/com/yelp/nrtsearch/server/nrt/NRTReplicaNode.java +++ b/src/main/java/com/yelp/nrtsearch/server/nrt/NRTReplicaNode.java @@ -159,7 +159,7 @@ protected CopyJob newCopyJob( } catch (Throwable t) { throw new NodeCommunicationException("exc while reading files to copy", t); } - files = copyState.files; + files = copyState.files(); } else { copyState = null; } @@ -250,7 +250,7 @@ protected void finishNRTCopy(CopyJob job, long startNS) throws IOException { .labelValues(indexName) .observe((System.nanoTime() - startNS) / 1000000.0); NrtMetrics.nrtPointSize.labelValues(indexName).observe(job.getTotalBytesCopied()); - NrtMetrics.searcherVersion.labelValues(indexName).set(job.getCopyState().version); + NrtMetrics.searcherVersion.labelValues(indexName).set(job.getCopyState().version()); } } @@ -356,7 +356,7 @@ public void syncFromCurrentPrimary(long primaryWaitMs, long maxTimeMs) throws IO logger.info( "Nrt sync: started new copy job, my version: {}, job version: {}", curVersion, - job.getCopyState().version); + job.getCopyState().version()); while (true) { curVersion = getCurrentSearchingVersion(); if (curVersion >= primaryIndexVersion) { diff --git a/src/main/java/com/yelp/nrtsearch/server/nrt/NrtDataManager.java b/src/main/java/com/yelp/nrtsearch/server/nrt/NrtDataManager.java index e03435f4d..bc0964a45 100644 --- a/src/main/java/com/yelp/nrtsearch/server/nrt/NrtDataManager.java +++ b/src/main/java/com/yelp/nrtsearch/server/nrt/NrtDataManager.java @@ -245,7 +245,7 @@ static UploadTask mergeTasks(UploadTask previous, UploadTask next, NRTPrimaryNod // make sure the latest version is used for the merged task CopyState taskCopyState; CopyState releaseCopyState; - if (previous.copyState.version <= next.copyState.version) { + if (previous.copyState.version() <= next.copyState.version()) { taskCopyState = next.copyState; releaseCopyState = previous.copyState; } else { @@ -294,7 +294,7 @@ public void run() { "Uploading new index files for service: {}, index: {}, version: {}", serviceName, indexIdentifier, - task.copyState.version); + task.copyState.version()); Map versionFiles = uploadDiff(task.copyState); NrtPointState pointState = new NrtPointState(task.copyState, versionFiles, ephemeralId); byte[] data = RemoteUtils.pointStateToUtf8(pointState); @@ -304,7 +304,7 @@ public void run() { logger.info( "Later version committed, skipping. Committed version: {}, Task version: {}", lastPointState.version, - task.copyState.version); + task.copyState.version()); } for (RefreshUploadFuture watcher : task.watchers) { watcher.setDone(null); @@ -330,7 +330,7 @@ public void run() { } private boolean isLaterVersion(CopyState copyState, NrtPointState lastPointState) { - return lastPointState == null || copyState.version > lastPointState.version; + return lastPointState == null || copyState.version() > lastPointState.version; } private Map uploadDiff(CopyState copyState) throws IOException { @@ -339,7 +339,7 @@ private Map uploadDiff(CopyState copyState) throws IOEx Map currentPointFiles = new HashMap<>(); Map filesToUpload = new HashMap<>(); - for (Map.Entry entry : copyState.files.entrySet()) { + for (Map.Entry entry : copyState.files().entrySet()) { String fileName = entry.getKey(); FileMetaData fileMetaData = entry.getValue(); @@ -361,10 +361,10 @@ private Map uploadDiff(CopyState copyState) throws IOEx @VisibleForTesting static boolean isSameFile(FileMetaData fileMetaData, NrtFileMetaData nrtFileMetaData) { - return fileMetaData.length == nrtFileMetaData.length - && fileMetaData.checksum == nrtFileMetaData.checksum - && Arrays.equals(fileMetaData.header, nrtFileMetaData.header) - && Arrays.equals(fileMetaData.footer, nrtFileMetaData.footer); + return fileMetaData.length() == nrtFileMetaData.length + && fileMetaData.checksum() == nrtFileMetaData.checksum + && Arrays.equals(fileMetaData.header(), nrtFileMetaData.header) + && Arrays.equals(fileMetaData.footer(), nrtFileMetaData.footer); } } diff --git a/src/main/java/com/yelp/nrtsearch/server/nrt/SimpleCopyJob.java b/src/main/java/com/yelp/nrtsearch/server/nrt/SimpleCopyJob.java index 3aa8feb99..85a4911f7 100644 --- a/src/main/java/com/yelp/nrtsearch/server/nrt/SimpleCopyJob.java +++ b/src/main/java/com/yelp/nrtsearch/server/nrt/SimpleCopyJob.java @@ -80,11 +80,11 @@ public void start() throws IOException { iter = toCopy.iterator(); // This means we resumed an already in-progress copy; we do this one first: if (current != null) { - totBytes += current.metaData.length; + totBytes += current.metaData.length(); } for (Map.Entry ent : toCopy) { FileMetaData metaData = ent.getValue(); - totBytes += metaData.length; + totBytes += metaData.length(); } // Send all file names / offsets up front to avoid ping-ping latency: diff --git a/src/main/java/com/yelp/nrtsearch/server/nrt/state/NrtFileMetaData.java b/src/main/java/com/yelp/nrtsearch/server/nrt/state/NrtFileMetaData.java index ac301f2fa..2190f6605 100644 --- a/src/main/java/com/yelp/nrtsearch/server/nrt/state/NrtFileMetaData.java +++ b/src/main/java/com/yelp/nrtsearch/server/nrt/state/NrtFileMetaData.java @@ -29,14 +29,18 @@ import org.apache.lucene.replicator.nrt.FileMetaData; /** - * Extension of {@link FileMetaData} that includes additional metadata for NRT replication, such as - * primaryId and timeString. + * Replacement for {@link FileMetaData} that includes additional metadata for NRT replication, such + * as primaryId and timeString. */ @JsonDeserialize(using = NrtFileMetaDataDeserializer.class) @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(JsonInclude.Include.NON_NULL) -public class NrtFileMetaData extends FileMetaData { +public class NrtFileMetaData { + public byte[] header; + public byte[] footer; + public long length; + public long checksum; public String primaryId; public String timeString; @@ -57,7 +61,10 @@ public NrtFileMetaData( long checksum, String primaryId, String timeString) { - super(header, footer, length, checksum); + this.header = header; + this.footer = footer; + this.length = length; + this.checksum = checksum; this.primaryId = primaryId; this.timeString = timeString; } @@ -70,7 +77,10 @@ public NrtFileMetaData( * @param timeString time string */ public NrtFileMetaData(FileMetaData metaData, String primaryId, String timeString) { - super(metaData.header, metaData.footer, metaData.length, metaData.checksum); + this.header = metaData.header(); + this.footer = metaData.footer(); + this.length = metaData.length(); + this.checksum = metaData.checksum(); this.primaryId = primaryId; this.timeString = timeString; } @@ -88,6 +98,10 @@ public boolean equals(Object obj) { return false; } + public FileMetaData toFileMetaData() { + return new FileMetaData(header, footer, length, checksum); + } + /** Custom json deserializer for NrtFileMetaData. */ public static class NrtFileMetaDataDeserializer extends StdDeserializer { diff --git a/src/main/java/com/yelp/nrtsearch/server/nrt/state/NrtPointState.java b/src/main/java/com/yelp/nrtsearch/server/nrt/state/NrtPointState.java index 8354936d1..fdecc6247 100644 --- a/src/main/java/com/yelp/nrtsearch/server/nrt/state/NrtPointState.java +++ b/src/main/java/com/yelp/nrtsearch/server/nrt/state/NrtPointState.java @@ -50,11 +50,11 @@ public NrtPointState() {} * @param primaryId primary id */ public NrtPointState(CopyState copyState, Map files, String primaryId) { - version = copyState.version; - gen = copyState.gen; - infosBytes = copyState.infosBytes; - primaryGen = copyState.primaryGen; - completedMergeFiles = copyState.completedMergeFiles; + version = copyState.version(); + gen = copyState.gen(); + infosBytes = copyState.infosBytes(); + primaryGen = copyState.primaryGen(); + completedMergeFiles = copyState.completedMergeFiles(); this.files = files; this.primaryId = primaryId; } @@ -67,7 +67,8 @@ public NrtPointState(CopyState copyState, Map files, St @JsonIgnore public CopyState toCopyState() { Map luceneFiles = - files.entrySet().stream().collect(Collectors.toMap(Entry::getKey, Entry::getValue)); + files.entrySet().stream() + .collect(Collectors.toMap(Entry::getKey, e -> e.getValue().toFileMetaData())); return new CopyState( luceneFiles, version, gen, infosBytes, completedMergeFiles, primaryGen, null); } diff --git a/src/main/java/com/yelp/nrtsearch/server/query/BlendedTermQuery.java b/src/main/java/com/yelp/nrtsearch/server/query/BlendedTermQuery.java index 2e0b74262..fa2aaaa9c 100644 --- a/src/main/java/com/yelp/nrtsearch/server/query/BlendedTermQuery.java +++ b/src/main/java/com/yelp/nrtsearch/server/query/BlendedTermQuery.java @@ -190,7 +190,7 @@ private TermStates adjustTTF( int df = termContext.docFreq(); long ttf = sumTTF; for (int i = 0; i < len; i++) { - TermState termState = termContext.get(leaves.get(i)); + TermState termState = termContext.get(leaves.get(i)).get(); if (termState == null) { continue; } @@ -215,7 +215,7 @@ private static TermStates adjustDF( } TermStates newCtx = new TermStates(readerContext); for (int i = 0; i < len; ++i) { - TermState termState = ctx.get(leaves.get(i)); + TermState termState = ctx.get(leaves.get(i)).get(); if (termState == null) { continue; } @@ -367,7 +367,7 @@ protected Query topLevelQuery(Term[] terms, TermStates[] ctx, int[] docFreqs, in if (low.clauses().isEmpty()) { BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder(); for (BooleanClause booleanClause : high) { - queryBuilder.add(booleanClause.getQuery(), Occur.MUST); + queryBuilder.add(booleanClause.query(), Occur.MUST); } return queryBuilder.build(); } else if (high.clauses().isEmpty()) { diff --git a/src/main/java/com/yelp/nrtsearch/server/query/QueryNodeMapper.java b/src/main/java/com/yelp/nrtsearch/server/query/QueryNodeMapper.java index 3480b315d..b70fe49eb 100644 --- a/src/main/java/com/yelp/nrtsearch/server/query/QueryNodeMapper.java +++ b/src/main/java/com/yelp/nrtsearch/server/query/QueryNodeMapper.java @@ -736,7 +736,7 @@ private static RegexpQuery getRegexpQuery( case REGEXP_ALL -> RegExp.ALL; case REGEXP_ANYSTRING -> RegExp.ANYSTRING; case REGEXP_AUTOMATON -> RegExp.AUTOMATON; - case REGEXP_COMPLEMENT -> RegExp.COMPLEMENT; + case REGEXP_COMPLEMENT -> RegExp.DEPRECATED_COMPLEMENT; case REGEXP_EMPTY -> RegExp.EMPTY; case REGEXP_INTERSECTION -> RegExp.INTERSECTION; case REGEXP_INTERVAL -> RegExp.INTERVAL; diff --git a/src/main/java/com/yelp/nrtsearch/server/query/QueryUtils.java b/src/main/java/com/yelp/nrtsearch/server/query/QueryUtils.java index 12c98dbe7..f5f00f49a 100644 --- a/src/main/java/com/yelp/nrtsearch/server/query/QueryUtils.java +++ b/src/main/java/com/yelp/nrtsearch/server/query/QueryUtils.java @@ -107,7 +107,7 @@ public double doubleValue() throws IOException { @Override public boolean advanceExact(int doc) throws IOException { - return scorer != null && scorer.docID() == doc; + return scorer != null; } } diff --git a/src/main/java/com/yelp/nrtsearch/server/query/multifunction/MultiFunctionScoreQuery.java b/src/main/java/com/yelp/nrtsearch/server/query/multifunction/MultiFunctionScoreQuery.java index 802e90bd4..27d7f3dd1 100644 --- a/src/main/java/com/yelp/nrtsearch/server/query/multifunction/MultiFunctionScoreQuery.java +++ b/src/main/java/com/yelp/nrtsearch/server/query/multifunction/MultiFunctionScoreQuery.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Objects; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.FilterScorer; @@ -142,7 +143,7 @@ public Query rewrite(IndexSearcher indexSearcher) throws IOException { @Override public void visit(QueryVisitor visitor) { - innerQuery.visit(visitor); + innerQuery.visit(visitor.getSubVisitor(BooleanClause.Occur.MUST, this)); } @Override @@ -256,6 +257,45 @@ public Explanation explain(LeafReaderContext context, int doc) throws IOExceptio return expl; } + @Override + public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { + ScorerSupplier innerScorerSupplier = innerWeight.scorerSupplier(context); + if (innerScorerSupplier == null) { + return null; + } + return new ScorerSupplier() { + @Override + public Scorer get(long leadCost) throws IOException { + Scorer innerScorer = innerWeight.scorer(context); + LeafFunction[] leafFunctions = new LeafFunction[functions.length]; + Bits[] docSets = new Bits[functions.length]; + for (int i = 0; i < filterWeights.length; ++i) { + leafFunctions[i] = functions[i].getLeafFunction(context); + if (filterWeights[i] != null) { + ScorerSupplier filterScorerSupplier = filterWeights[i].scorerSupplier(context); + docSets[i] = + QueryUtils.asSequentialAccessBits( + context.reader().maxDoc(), filterScorerSupplier); + } else { + docSets[i] = new Bits.MatchAllBits(context.reader().maxDoc()); + } + } + + Scorer scorer = + new MultiFunctionScorer(innerScorer, scoreMode, boostMode, leafFunctions, docSets); + if (isMinScoreWrapperUsed()) { + scorer = new MinScoreWrapper(scorer, minScore, minExcluded); + } + return scorer; + } + + @Override + public long cost() { + return innerScorerSupplier.cost(); + } + }; + } + private Explanation explainBoost(Explanation queryExpl, Explanation funcExpl) { return switch (boostMode) { case BOOST_MODE_MULTIPLY -> @@ -280,38 +320,9 @@ private Explanation explainBoost(Explanation queryExpl, Explanation funcExpl) { }; } - @Override - public Scorer scorer(LeafReaderContext context) throws IOException { - Scorer innerScorer = innerWeight.scorer(context); - if (innerScorer == null) { - return null; - } - - LeafFunction[] leafFunctions = new LeafFunction[functions.length]; - Bits[] docSets = new Bits[functions.length]; - for (int i = 0; i < filterWeights.length; ++i) { - leafFunctions[i] = functions[i].getLeafFunction(context); - if (filterWeights[i] != null) { - ScorerSupplier filterScorerSupplier = filterWeights[i].scorerSupplier(context); - docSets[i] = - QueryUtils.asSequentialAccessBits(context.reader().maxDoc(), filterScorerSupplier); - } else { - docSets[i] = new Bits.MatchAllBits(context.reader().maxDoc()); - } - } - - Scorer scorer = - new MultiFunctionScorer(innerScorer, this, scoreMode, boostMode, leafFunctions, docSets); - if (isMinScoreWrapperUsed()) { - scorer = new MinScoreWrapper(scorer.getWeight(), scorer, minScore, minExcluded); - } - return scorer; - } - @Override public boolean isCacheable(LeafReaderContext ctx) { - // When not using MinScoreWrapper, it is cacheable. - return !isMinScoreWrapperUsed(); + return false; } } @@ -335,8 +346,7 @@ public static class MinScoreWrapper extends Scorer { private float curScore; private final boolean minExcluded; - public MinScoreWrapper(Weight weight, Scorer in, float minScore, boolean minExcluded) { - super(weight); + public MinScoreWrapper(Scorer in, float minScore, boolean minExcluded) { this.in = in; this.minScore = minScore; this.minExcluded = minExcluded; @@ -415,12 +425,11 @@ public static class MultiFunctionScorer extends FilterScorer { public MultiFunctionScorer( Scorer innerScorer, - MultiFunctionWeight weight, FunctionScoreMode scoreMode, BoostMode boostMode, LeafFunction[] leafFunctions, Bits[] docSets) { - super(innerScorer, weight); + super(innerScorer); this.scoreMode = scoreMode; this.boostMode = boostMode; this.leafFunctions = leafFunctions; diff --git a/src/main/java/com/yelp/nrtsearch/server/query/vector/ExactVectorQuery.java b/src/main/java/com/yelp/nrtsearch/server/query/vector/ExactVectorQuery.java index 06159a53f..a207347e9 100644 --- a/src/main/java/com/yelp/nrtsearch/server/query/vector/ExactVectorQuery.java +++ b/src/main/java/com/yelp/nrtsearch/server/query/vector/ExactVectorQuery.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; import org.apache.lucene.search.VectorScorer; import org.apache.lucene.search.Weight; @@ -74,35 +75,59 @@ public String getField() { @Override public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) { - return new Weight(this) { - @Override - public Explanation explain(LeafReaderContext context, int doc) throws IOException { - VectorScorer vectorScorer = vectorScorerFunction.get(context.reader()); - if (vectorScorer == null) { - return Explanation.noMatch("No vector found for field: " + field); - } - DocIdSetIterator iterator = vectorScorer.iterator(); - if (iterator.advance(doc) == doc) { - float score = vectorScorer.score(); - return Explanation.match(score, "Found vector with similarity: " + score); - } - return Explanation.noMatch("No document vector for field: " + field); - } + return new ExactVectorQueryWeight(this, field, boost, vectorScorerFunction); + } + + private static class ExactVectorQueryWeight extends Weight { + private final String field; + private final float boost; + private final VectorScorerSupplier vectorScorerFunction; + + /** + * Sole constructor. + * + * @param query the parent query + * @param field the field to search + * @param boost scoring boost + * @param vectorScorerFunction function for creating a {@link VectorScorer} for a given {@link + * LeafReader} + */ + protected ExactVectorQueryWeight( + Query query, String field, float boost, VectorScorerSupplier vectorScorerFunction) { + super(query); + this.field = field; + this.boost = boost; + this.vectorScorerFunction = vectorScorerFunction; + } - @Override - public Scorer scorer(LeafReaderContext context) throws IOException { - VectorScorer vectorScorer = vectorScorerFunction.get(context.reader()); - if (vectorScorer == null) { - return null; - } - return new VectorValuesScorer(this, vectorScorer); + @Override + public Explanation explain(LeafReaderContext context, int doc) throws IOException { + VectorScorer vectorScorer = vectorScorerFunction.get(context.reader()); + if (vectorScorer == null) { + return Explanation.noMatch("No vector found for field: " + field); + } + DocIdSetIterator iterator = vectorScorer.iterator(); + if (iterator.advance(doc) == doc) { + float score = vectorScorer.score(); + return Explanation.match(score * boost, "Found vector with similarity: " + score); } + return Explanation.noMatch("No document vector for field: " + field); + } - @Override - public boolean isCacheable(LeafReaderContext ctx) { - return true; + @Override + public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { + VectorScorer vectorScorer = vectorScorerFunction.get(context.reader()); + if (vectorScorer == null) { + return null; } - }; + Scorer vectorValuesScorer = new VectorValuesScorer(vectorScorer, boost); + return new DefaultScorerSupplier(vectorValuesScorer); + } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return true; + } } /** @@ -112,17 +137,18 @@ public boolean isCacheable(LeafReaderContext ctx) { private static class VectorValuesScorer extends Scorer { private final VectorScorer vectorScorer; private final DocIdSetIterator iterator; + private final float boost; /** * Constructor. * - * @param weight the weight that created the scorer * @param vectorScorer the vector scorer to use + * @param boost scoring boost */ - protected VectorValuesScorer(Weight weight, VectorScorer vectorScorer) { - super(weight); + protected VectorValuesScorer(VectorScorer vectorScorer, float boost) { this.vectorScorer = vectorScorer; this.iterator = vectorScorer.iterator(); + this.boost = boost; } @Override @@ -137,7 +163,7 @@ public float getMaxScore(int upTo) throws IOException { @Override public float score() throws IOException { - return vectorScorer.score(); + return vectorScorer.score() * boost; } @Override diff --git a/src/main/java/com/yelp/nrtsearch/server/rescore/QueryRescore.java b/src/main/java/com/yelp/nrtsearch/server/rescore/QueryRescore.java index c226dba82..8ef35105b 100644 --- a/src/main/java/com/yelp/nrtsearch/server/rescore/QueryRescore.java +++ b/src/main/java/com/yelp/nrtsearch/server/rescore/QueryRescore.java @@ -51,7 +51,7 @@ public static Builder newBuilder() { @Override public TopDocs rescore(TopDocs hits, RescoreContext context) throws IOException { return rescore( - context.getSearchContext().getSearcherAndTaxonomy().searcher, + context.getSearchContext().getSearcherAndTaxonomy().searcher(), hits, context.getWindowSize()); } diff --git a/src/main/java/com/yelp/nrtsearch/server/search/MyIndexSearcher.java b/src/main/java/com/yelp/nrtsearch/server/search/MyIndexSearcher.java index 85134de95..fe39cf8bd 100644 --- a/src/main/java/com/yelp/nrtsearch/server/search/MyIndexSearcher.java +++ b/src/main/java/com/yelp/nrtsearch/server/search/MyIndexSearcher.java @@ -15,75 +15,48 @@ */ package com.yelp.nrtsearch.server.search; -import static org.apache.lucene.facet.DrillSidewaysQueryCheck.isDrillSidewaysQuery; - -import java.io.IOException; +import com.google.common.annotations.VisibleForTesting; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.Objects; import java.util.PriorityQueue; import java.util.concurrent.Executor; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.BulkScorer; -import org.apache.lucene.search.CollectionTerminatedException; -import org.apache.lucene.search.Collector; -import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.LeafCollector; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Weight; -import org.apache.lucene.search.suggest.document.CompletionQuery; -import org.apache.lucene.util.Bits; /** - * This is sadly necessary because for ToParentBlockJoinQuery we must invoke .scorer not - * .bulkScorer, yet for DrillSideways we must do exactly the opposite! + * Custom IndexSearcher that allows for custom slicing of the index into multiple segments for + * parallel search. */ public class MyIndexSearcher extends IndexSearcher { - /** - * Class that uses an Executor implementation to hold the parallel search Executor and any - * parameters needed to compute index search slices. This is hacky, but unfortunately necessary - * since {@link IndexSearcher#slices(List)} is called directly from the constructor, which happens - * before any member variable are set in the child class. - */ - public static class ExecutorWithParams implements Executor { - final Executor wrapped; - final int sliceMaxDocs; - final int sliceMaxSegments; - final int virtualShards; + public record SlicingParams(int sliceMaxDocs, int sliceMaxSegments, int virtualShards) {} - /** - * Constructor. - * - * @param wrapped executor to perform parallel search operations - * @param sliceMaxDocs max docs per index slice - * @param sliceMaxSegments max segments per index slice - * @param virtualShards number for virtual shards for index - * @throws NullPointerException if wrapped is null - */ - public ExecutorWithParams( - Executor wrapped, int sliceMaxDocs, int sliceMaxSegments, int virtualShards) { - Objects.requireNonNull(wrapped); - this.wrapped = wrapped; - this.sliceMaxDocs = sliceMaxDocs; - this.sliceMaxSegments = sliceMaxSegments; - this.virtualShards = virtualShards; - } + private static final Object slicingLock = new Object(); + private static SlicingParams staticSlicingParams; - @Override - public void execute(Runnable command) { - wrapped.execute(command); - } + private SlicingParams slicingParams; - @Override - public String toString() { - return String.format( - "ExecutorWithParams(sliceMaxDocs=%d, sliceMaxSegments=%d, virtualShards=%d, wrapped=%s)", - sliceMaxDocs, sliceMaxSegments, virtualShards, wrapped); + /** + * Create a new MyIndexSearcher. + * + * @param reader index reader + * @param executor parallel search task executor + * @param slicingParams slicing parameters + * @return MyIndexSearcher + */ + public static MyIndexSearcher create( + IndexReader reader, Executor executor, SlicingParams slicingParams) { + // Use lock to serialize initialization of index searchers. The slicing params can only be + // passed in the static + // context. To allow for different configuration per index, we use a single static variable to + // hold the slicing + // params, which is read only during construction of the index searcher. + synchronized (slicingLock) { + staticSlicingParams = slicingParams; + return new MyIndexSearcher(reader, executor); } } @@ -91,26 +64,31 @@ public String toString() { * Constructor. * * @param reader index reader - * @param executorWithParams parameter class that hold search executor and slice config + * @param executor parallel search task executor */ - public MyIndexSearcher(IndexReader reader, ExecutorWithParams executorWithParams) { - super(reader, executorWithParams); + protected MyIndexSearcher(IndexReader reader, Executor executor) { + super(reader, executor); + } + + @VisibleForTesting + SlicingParams getSlicingParams() { + return slicingParams; } /** * start segment to thread mapping * */ protected LeafSlice[] slices(List leaves) { - if (!(getExecutor() instanceof ExecutorWithParams)) { - throw new IllegalArgumentException("Executor must be an ExecutorWithParams"); + slicingParams = staticSlicingParams; + if (slicingParams == null) { + throw new IllegalArgumentException("Slicing params not set"); } - ExecutorWithParams executorWithParams = (ExecutorWithParams) getExecutor(); - if (executorWithParams.virtualShards > 1) { + if (slicingParams.virtualShards > 1) { return slicesForShards( leaves, - executorWithParams.virtualShards, - executorWithParams.sliceMaxDocs, - executorWithParams.sliceMaxSegments); + slicingParams.virtualShards, + slicingParams.sliceMaxDocs, + slicingParams.sliceMaxSegments); } else { - return slices(leaves, executorWithParams.sliceMaxDocs, executorWithParams.sliceMaxSegments); + return slices(leaves, slicingParams.sliceMaxDocs, slicingParams.sliceMaxSegments); } } @@ -132,9 +110,7 @@ private static class SliceAndSize { SliceAndSize(LeafSlice slice) { this.slice = slice; - for (int i = 0; i < slice.leaves.length; ++i) { - numDocs += slice.leaves[i].reader().numDocs(); - } + numDocs = slice.getMaxDocs(); } } @@ -183,12 +159,6 @@ private static LeafSlice[] slicesForShards( return slices; } - /* Better Segment To Thread Mapping Algorithm: https://issues.apache.org/jira/browse/LUCENE-8757 - This change is available in 9.0 (master) which is not released yet - https://github.com/apache/lucene-solr/blob/master/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java#L316 - We can remove this method once luceneVersion is updated to 9.x - * */ - /** Static method to segregate LeafReaderContexts amongst multiple slices */ public static LeafSlice[] slices( List leaves, int maxDocsPerSlice, int maxSegmentsPerSlice) { @@ -199,21 +169,22 @@ public static LeafSlice[] slices( Collections.sort( sortedLeaves, Collections.reverseOrder(Comparator.comparingInt(l -> l.reader().maxDoc()))); - final List> groupedLeaves = new ArrayList<>(); + final List> groupedLeaves = new ArrayList<>(); long docSum = 0; - List group = null; + List group = null; for (LeafReaderContext ctx : sortedLeaves) { if (ctx.reader().maxDoc() > maxDocsPerSlice) { assert group == null; - groupedLeaves.add(Collections.singletonList(ctx)); + groupedLeaves.add( + Collections.singletonList(LeafReaderContextPartition.createForEntireSegment(ctx))); } else { if (group == null) { group = new ArrayList<>(); - group.add(ctx); + group.add(LeafReaderContextPartition.createForEntireSegment(ctx)); groupedLeaves.add(group); } else { - group.add(ctx); + group.add(LeafReaderContextPartition.createForEntireSegment(ctx)); } docSum += ctx.reader().maxDoc(); @@ -226,66 +197,13 @@ public static LeafSlice[] slices( LeafSlice[] slices = new LeafSlice[groupedLeaves.size()]; int upto = 0; - for (List currentLeaf : groupedLeaves) { + for (List currentLeaf : groupedLeaves) { // LeafSlice constructor has changed in 9.x. This allows to use old constructor. - Collections.sort(currentLeaf, Comparator.comparingInt(l -> l.docBase)); + Collections.sort(currentLeaf, Comparator.comparingInt(l -> l.ctx.docBase)); slices[upto] = new LeafSlice(currentLeaf); ++upto; } return slices; } - - /** * end segment to thread mapping * */ - @Override - protected void search(List leaves, Weight weight, Collector collector) - throws IOException { - boolean isDrillSidewaysQueryOrCompletionQuery = - weight.getQuery() instanceof CompletionQuery || isDrillSidewaysQuery(weight.getQuery()); - for (LeafReaderContext ctx : leaves) { // search each subreader - // we force the use of Scorer (not BulkScorer) to make sure - // that the scorer passed to LeafCollector.setScorer supports - // Scorer.getChildren - final LeafCollector leafCollector; - try { - leafCollector = collector.getLeafCollector(ctx); - } catch (CollectionTerminatedException e) { - // there is no doc of interest in this reader context - // continue with the following leaf - continue; - } - if (isDrillSidewaysQueryOrCompletionQuery) { - BulkScorer scorer = weight.bulkScorer(ctx); - if (scorer != null) { - try { - scorer.score( - leafCollector, ctx.reader().getLiveDocs(), 0, DocIdSetIterator.NO_MORE_DOCS); - } catch (CollectionTerminatedException e) { - // collection was terminated prematurely - // continue with the following leaf - } - } - } else { - Scorer scorer = weight.scorer(ctx); - if (scorer != null) { - leafCollector.setScorer(scorer); - final Bits liveDocs = ctx.reader().getLiveDocs(); - final DocIdSetIterator it = scorer.iterator(); - try { - for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) { - if (liveDocs == null || liveDocs.get(doc)) { - leafCollector.collect(doc); - } - } - } catch (CollectionTerminatedException e) { - // collection was terminated prematurely - // continue with the following leaf - } - } - } - // Note: this is called if collection ran successfully, including the above special case of - // CollectionTerminatedException, but no other exception. - leafCollector.finish(); - } - } } diff --git a/src/main/java/com/yelp/nrtsearch/server/search/SearchRequestProcessor.java b/src/main/java/com/yelp/nrtsearch/server/search/SearchRequestProcessor.java index 3d4cad04b..034807cf1 100644 --- a/src/main/java/com/yelp/nrtsearch/server/search/SearchRequestProcessor.java +++ b/src/main/java/com/yelp/nrtsearch/server/search/SearchRequestProcessor.java @@ -162,7 +162,7 @@ public static SearchContext buildContextForRequest( profileResult.setParsedQuery(query.toString()); } - query = searcherAndTaxonomy.searcher.rewrite(query); + query = searcherAndTaxonomy.searcher().rewrite(query); if (profileResult != null) { profileResult.setRewrittenQuery(query.toString()); } @@ -213,7 +213,7 @@ public static SearchContext buildContextForRequest( contextBuilder.setCollector(docCollector); contextBuilder.setRescorers( - getRescorers(indexState, searcherAndTaxonomy.searcher, searchRequest)); + getRescorers(indexState, searcherAndTaxonomy.searcher(), searchRequest)); contextBuilder.setSharedDocContext(new DefaultSharedDocContext()); contextBuilder.setExtraContext(new ConcurrentHashMap<>()); @@ -237,7 +237,7 @@ public static SearchContext buildContextForRequest( for (int i = 0; i < knnQueries.size(); ++i) { Query resolvedKnnQuery = resolveKnnQueryAndBoost( - knnQueries.get(i), knnBoosts.get(i), searcherAndTaxonomy.searcher, diagnostics); + knnQueries.get(i), knnBoosts.get(i), searcherAndTaxonomy.searcher(), diagnostics); queryBuilder.add(resolvedKnnQuery, BooleanClause.Occur.SHOULD); } query = queryBuilder.build(); @@ -332,8 +332,8 @@ private static Query resolveKnnQueryAndBoost( com.yelp.nrtsearch.server.grpc.TotalHits.newBuilder() .setRelation( com.yelp.nrtsearch.server.grpc.TotalHits.Relation.valueOf( - vectorTotalHits.relation.name())) - .setValue(vectorTotalHits.value) + vectorTotalHits.relation().name())) + .setValue(vectorTotalHits.value()) .build()); } diagnostics.addVectorDiagnostics(vectorDiagnosticsBuilder.build()); diff --git a/src/main/java/com/yelp/nrtsearch/server/search/collectors/additional/FilterCollectorManager.java b/src/main/java/com/yelp/nrtsearch/server/search/collectors/additional/FilterCollectorManager.java index c5fe707f0..fd9b002a0 100644 --- a/src/main/java/com/yelp/nrtsearch/server/search/collectors/additional/FilterCollectorManager.java +++ b/src/main/java/com/yelp/nrtsearch/server/search/collectors/additional/FilterCollectorManager.java @@ -95,11 +95,11 @@ private static class QueryFilter implements Filter { QueryNodeMapper.getInstance().getQuery(grpcQuery, context.getIndexState()); try { org.apache.lucene.search.Query rewritten = - context.getSearcherAndTaxonomy().searcher.rewrite(query); + context.getSearcherAndTaxonomy().searcher().rewrite(query); filterWeight = context .getSearcherAndTaxonomy() - .searcher + .searcher() .createWeight(rewritten, ScoreMode.COMPLETE_NO_SCORES, 1.0f); } catch (IOException e) { throw new RuntimeException("Error creating filter query weight", e); diff --git a/src/main/java/com/yelp/nrtsearch/server/search/collectors/additional/OrdinalTermsCollectorManager.java b/src/main/java/com/yelp/nrtsearch/server/search/collectors/additional/OrdinalTermsCollectorManager.java index 7d048ad43..8b4d5b28b 100644 --- a/src/main/java/com/yelp/nrtsearch/server/search/collectors/additional/OrdinalTermsCollectorManager.java +++ b/src/main/java/com/yelp/nrtsearch/server/search/collectors/additional/OrdinalTermsCollectorManager.java @@ -77,7 +77,7 @@ public OrdinalTermsCollectorManager( try { globalOrdinalLookup = globalOrdinalable.getOrdinalLookup( - context.getSearcherAndTaxonomy().searcher.getIndexReader()); + context.getSearcherAndTaxonomy().searcher().getIndexReader()); } catch (IOException e) { throw new RuntimeException("Error getting ordinal map"); } diff --git a/src/main/java/com/yelp/nrtsearch/server/search/collectors/additional/TopHitsCollectorManager.java b/src/main/java/com/yelp/nrtsearch/server/search/collectors/additional/TopHitsCollectorManager.java index 4287b6f25..d6db0d0a0 100644 --- a/src/main/java/com/yelp/nrtsearch/server/search/collectors/additional/TopHitsCollectorManager.java +++ b/src/main/java/com/yelp/nrtsearch/server/search/collectors/additional/TopHitsCollectorManager.java @@ -165,8 +165,8 @@ public CollectorResult reduce(Collection collectors) throws IO TotalHits totalHits = TotalHits.newBuilder() - .setRelation(TotalHits.Relation.valueOf(topDocs.totalHits.relation.name())) - .setValue(topDocs.totalHits.value) + .setRelation(TotalHits.Relation.valueOf(topDocs.totalHits.relation().name())) + .setValue(topDocs.totalHits.value()) .build(); hitsResultBuilder.setTotalHits(totalHits); for (int hitIndex = 0; hitIndex < topDocs.scoreDocs.length; hitIndex++) { @@ -194,7 +194,7 @@ public CollectorResult reduce(Collection collectors) throws IO hitBuilder.setExplain( searchContext .getSearcherAndTaxonomy() - .searcher + .searcher() .explain(searchContext.getQuery(), hitBuilder.getLuceneDocId()) .toString()); } catch (IOException ioException) { diff --git a/src/main/java/org/apache/lucene/replicator/nrt/CopyOneFile.java b/src/main/java/org/apache/lucene/replicator/nrt/CopyOneFile.java index 25de7455f..d52246858 100644 --- a/src/main/java/org/apache/lucene/replicator/nrt/CopyOneFile.java +++ b/src/main/java/org/apache/lucene/replicator/nrt/CopyOneFile.java @@ -54,7 +54,7 @@ public CopyOneFile( out = dest.createTempOutput(name, "copy", IOContext.DEFAULT); tmpName = out.getName(); // last 8 bytes are checksum: - bytesToCopy = metaData.length - Long.BYTES; + bytesToCopy = metaData.length() - Long.BYTES; if (Node.VERBOSE_FILES) { dest.message( "file " @@ -123,7 +123,7 @@ public boolean visit() throws IOException { return false; } else { long checksum = out.getChecksum(); - if (checksum != metaData.checksum) { + if (checksum != metaData.checksum()) { // Bits flipped during copy! dest.message( "file " @@ -131,7 +131,7 @@ public boolean visit() throws IOException { + ": checksum mismatch after copy (bits flipped during network copy?) after-copy checksum=" + checksum + " vs expected=" - + metaData.checksum + + metaData.checksum() + "; cancel job"); throw new IOException("file " + name + ": checksum mismatch after file copy"); } @@ -157,7 +157,7 @@ public boolean visit() throws IOException { Locale.ROOT, "file %s: done copying [%s, %.3fms]", name, - Node.bytesToString(metaData.length), + Node.bytesToString(metaData.length()), (System.nanoTime() - copyStartNS) / 1000000.0)); } return true; diff --git a/src/test/java/com/yelp/nrtsearch/server/codec/ServerCodecTest.java b/src/test/java/com/yelp/nrtsearch/server/codec/ServerCodecTest.java index e541700d6..10c6250e0 100644 --- a/src/test/java/com/yelp/nrtsearch/server/codec/ServerCodecTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/codec/ServerCodecTest.java @@ -32,8 +32,8 @@ import org.apache.lucene.backward_codecs.lucene80.Lucene80DocValuesFormat; import org.apache.lucene.backward_codecs.lucene90.Lucene90HnswVectorsFormat; import org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat; +import org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat; import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; -import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat; import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.junit.BeforeClass; import org.junit.Test; @@ -61,7 +61,7 @@ public void testPostingFormat_default() { when(mockFieldDef.getPostingsFormat()).thenReturn(null); IndexStateManager mockStateManager = getManager(mockFieldDef); ServerCodec serverCodec = new ServerCodec(mockStateManager); - assertTrue(serverCodec.getPostingsFormatForField("field") instanceof Lucene912PostingsFormat); + assertTrue(serverCodec.getPostingsFormatForField("field") instanceof Lucene101PostingsFormat); verify(mockFieldDef, times(1)).getPostingsFormat(); verifyNoMoreInteractions(mockFieldDef); } @@ -97,7 +97,7 @@ public void testPostingFormat_internalField() { IndexStateManager mockStateManager = getManager(mockFieldDef); ServerCodec serverCodec = new ServerCodec(mockStateManager); assertTrue( - serverCodec.getPostingsFormatForField("internal_field") instanceof Lucene912PostingsFormat); + serverCodec.getPostingsFormatForField("internal_field") instanceof Lucene101PostingsFormat); verifyNoInteractions(mockFieldDef); } diff --git a/src/test/java/com/yelp/nrtsearch/server/doc/SharedDocContextTest.java b/src/test/java/com/yelp/nrtsearch/server/doc/SharedDocContextTest.java index 04c0bb582..6e88b65a6 100644 --- a/src/test/java/com/yelp/nrtsearch/server/doc/SharedDocContextTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/doc/SharedDocContextTest.java @@ -133,7 +133,7 @@ public void processHit( SearchContext searchContext, LeafReaderContext hitLeaf, SearchResponse.Hit.Builder hit) throws IOException { List leaves = - searchContext.getSearcherAndTaxonomy().searcher.getIndexReader().leaves(); + searchContext.getSearcherAndTaxonomy().searcher().getIndexReader().leaves(); int leafIndex = ReaderUtil.subIndex(hit.getLuceneDocId(), leaves); LeafReaderContext leaf = leaves.get(leafIndex); SegmentDocLookup segmentDocLookup = diff --git a/src/test/java/com/yelp/nrtsearch/server/facet/EagerGlobalOrdinalsTest.java b/src/test/java/com/yelp/nrtsearch/server/facet/EagerGlobalOrdinalsTest.java index d84db5dd7..94538ff40 100644 --- a/src/test/java/com/yelp/nrtsearch/server/facet/EagerGlobalOrdinalsTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/facet/EagerGlobalOrdinalsTest.java @@ -86,16 +86,16 @@ public void testEagerOrdinals() throws Exception { getGlobalState().getIndexOrThrow(DEFAULT_TEST_INDEX).getFieldOrThrow(EAGER_FIELD); addDocAndRefresh(); s = shardState.acquire(); - assertGlobalOrdinals(s.searcher.getIndexReader(), fieldDef); + assertGlobalOrdinals(s.searcher().getIndexReader(), fieldDef); doQuery(EAGER_FIELD); - assertGlobalOrdinals(s.searcher.getIndexReader(), fieldDef); + assertGlobalOrdinals(s.searcher().getIndexReader(), fieldDef); // create new reader version addDocAndRefresh(); shardState.release(s); s = null; s = shardState.acquire(); - assertGlobalOrdinals(s.searcher.getIndexReader(), fieldDef); + assertGlobalOrdinals(s.searcher().getIndexReader(), fieldDef); } finally { if (s != null) { shardState.release(s); @@ -112,18 +112,18 @@ public void testWithoutEagerOrdinals() throws Exception { getGlobalState().getIndexOrThrow(DEFAULT_TEST_INDEX).getFieldOrThrow(NOT_EAGER_FIELD); addDocAndRefresh(); s = shardState.acquire(); - assertNoGlobalOrdinals(s.searcher.getIndexReader(), fieldDef); + assertNoGlobalOrdinals(s.searcher().getIndexReader(), fieldDef); doQuery(NOT_EAGER_FIELD); - assertGlobalOrdinals(s.searcher.getIndexReader(), fieldDef); + assertGlobalOrdinals(s.searcher().getIndexReader(), fieldDef); // create new reader version addDocAndRefresh(); shardState.release(s); s = null; s = shardState.acquire(); - assertNoGlobalOrdinals(s.searcher.getIndexReader(), fieldDef); + assertNoGlobalOrdinals(s.searcher().getIndexReader(), fieldDef); doQuery(NOT_EAGER_FIELD); - assertGlobalOrdinals(s.searcher.getIndexReader(), fieldDef); + assertGlobalOrdinals(s.searcher().getIndexReader(), fieldDef); } finally { if (s != null) { shardState.release(s); diff --git a/src/test/java/com/yelp/nrtsearch/server/field/EagerFieldOrdinalTest.java b/src/test/java/com/yelp/nrtsearch/server/field/EagerFieldOrdinalTest.java index 96d577cc2..6d9a34d1b 100644 --- a/src/test/java/com/yelp/nrtsearch/server/field/EagerFieldOrdinalTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/field/EagerFieldOrdinalTest.java @@ -129,16 +129,16 @@ public void testEagerOrdinals() throws Exception { getGlobalState().getIndexOrThrow(DEFAULT_TEST_INDEX).getFieldOrThrow(field); addDocAndRefresh(); s = shardState.acquire(); - assertGlobalOrdinals(s.searcher.getIndexReader(), fieldDef); + assertGlobalOrdinals(s.searcher().getIndexReader(), fieldDef); doQuery(field); - assertGlobalOrdinals(s.searcher.getIndexReader(), fieldDef); + assertGlobalOrdinals(s.searcher().getIndexReader(), fieldDef); // create new reader version addDocAndRefresh(); shardState.release(s); s = null; s = shardState.acquire(); - assertGlobalOrdinals(s.searcher.getIndexReader(), fieldDef); + assertGlobalOrdinals(s.searcher().getIndexReader(), fieldDef); } } finally { if (s != null) { @@ -157,18 +157,18 @@ public void testWithoutEagerOrdinals() throws Exception { getGlobalState().getIndexOrThrow(DEFAULT_TEST_INDEX).getFieldOrThrow(field); addDocAndRefresh(); s = shardState.acquire(); - assertNoGlobalOrdinals(s.searcher.getIndexReader(), fieldDef); + assertNoGlobalOrdinals(s.searcher().getIndexReader(), fieldDef); doQuery(field); - assertGlobalOrdinals(s.searcher.getIndexReader(), fieldDef); + assertGlobalOrdinals(s.searcher().getIndexReader(), fieldDef); // create new reader version addDocAndRefresh(); shardState.release(s); s = null; s = shardState.acquire(); - assertNoGlobalOrdinals(s.searcher.getIndexReader(), fieldDef); + assertNoGlobalOrdinals(s.searcher().getIndexReader(), fieldDef); doQuery(field); - assertGlobalOrdinals(s.searcher.getIndexReader(), fieldDef); + assertGlobalOrdinals(s.searcher().getIndexReader(), fieldDef); } } finally { if (s != null) { diff --git a/src/test/java/com/yelp/nrtsearch/server/field/VectorFieldDefTest.java b/src/test/java/com/yelp/nrtsearch/server/field/VectorFieldDefTest.java index d75d89890..f269176ee 100644 --- a/src/test/java/com/yelp/nrtsearch/server/field/VectorFieldDefTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/field/VectorFieldDefTest.java @@ -51,6 +51,7 @@ import org.apache.lucene.index.ByteVectorValues; import org.apache.lucene.index.FloatVectorValues; import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.KnnVectorValues; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.VectorSimilarityFunction; @@ -2143,20 +2144,22 @@ private List getTrueFloatTopHits( PriorityQueue topHits = new PriorityQueue<>(5, Comparator.comparing(VectorSearchResult::score).reversed()); try { - List leaves = searcherAndTaxonomy.searcher.getLeafContexts(); + List leaves = searcherAndTaxonomy.searcher().getLeafContexts(); IndexableFieldDef filterField = (IndexableFieldDef) indexState.getField("filter"); float[] queryVectorArray = floatListToArray(queryVector); for (LeafReaderContext leaf : leaves) { FloatVectorValues vectorValues = leaf.reader().getFloatVectorValues(field); + KnnVectorValues.DocIndexIterator docIndexIterator = vectorValues.iterator(); LoadedDocValues filterValues = (LoadedDocValues) filterField.getDocValues(leaf); for (int i = 0; i < leaf.reader().maxDoc(); ++i) { filterValues.setDocId(i); - if (vectorValues.advance(i) == i) { + if (docIndexIterator.advance(i) == i) { if (filter == null || filter.equals(filterValues.getFirst())) { float similarity = - similarityFunction.compare(queryVectorArray, vectorValues.vectorValue()); + similarityFunction.compare( + queryVectorArray, vectorValues.vectorValue(docIndexIterator.index())); topHits.add(new VectorSearchResult(leaf.docBase + i, similarity)); } } @@ -2191,19 +2194,21 @@ private List getTrueByteTopHits( PriorityQueue topHits = new PriorityQueue<>(5, Comparator.comparing(VectorSearchResult::score).reversed()); try { - List leaves = searcherAndTaxonomy.searcher.getLeafContexts(); + List leaves = searcherAndTaxonomy.searcher().getLeafContexts(); IndexableFieldDef filterField = (IndexableFieldDef) indexState.getField("filter"); for (LeafReaderContext leaf : leaves) { ByteVectorValues vectorValues = leaf.reader().getByteVectorValues(field); + KnnVectorValues.DocIndexIterator docIndexIterator = vectorValues.iterator(); LoadedDocValues filterValues = (LoadedDocValues) filterField.getDocValues(leaf); for (int i = 0; i < leaf.reader().maxDoc(); ++i) { filterValues.setDocId(i); - if (vectorValues.advance(i) == i) { + if (docIndexIterator.advance(i) == i) { if (filter == null || filter.equals(filterValues.getFirst())) { float similarity = - similarityFunction.compare(queryVector, vectorValues.vectorValue()); + similarityFunction.compare( + queryVector, vectorValues.vectorValue(docIndexIterator.index())); topHits.add(new VectorSearchResult(leaf.docBase + i, similarity)); } } diff --git a/src/test/java/com/yelp/nrtsearch/server/grpc/MultiSegmentTest.java b/src/test/java/com/yelp/nrtsearch/server/grpc/MultiSegmentTest.java index 0556085db..a8865d61d 100644 --- a/src/test/java/com/yelp/nrtsearch/server/grpc/MultiSegmentTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/grpc/MultiSegmentTest.java @@ -97,8 +97,8 @@ public void testNumSegments() throws Exception { SearcherTaxonomyManager.SearcherAndTaxonomy s = null; try { s = getGlobalState().getIndexOrThrow(TEST_INDEX).getShard(0).acquire(); - assertEquals(NUM_DOCS / SEGMENT_CHUNK, s.searcher.getIndexReader().leaves().size()); - for (LeafReaderContext context : s.searcher.getIndexReader().leaves()) { + assertEquals(NUM_DOCS / SEGMENT_CHUNK, s.searcher().getIndexReader().leaves().size()); + for (LeafReaderContext context : s.searcher().getIndexReader().leaves()) { assertEquals(SEGMENT_CHUNK, context.reader().maxDoc()); } } finally { diff --git a/src/test/java/com/yelp/nrtsearch/server/grpc/TimeoutTest.java b/src/test/java/com/yelp/nrtsearch/server/grpc/TimeoutTest.java index a9d22e57a..4e11e69c5 100644 --- a/src/test/java/com/yelp/nrtsearch/server/grpc/TimeoutTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/grpc/TimeoutTest.java @@ -153,8 +153,8 @@ public void testNumSegments() throws Exception { SearcherTaxonomyManager.SearcherAndTaxonomy s = null; try { s = getGlobalState().getIndexOrThrow(TEST_INDEX).getShard(0).acquire(); - assertEquals(NUM_DOCS / SEGMENT_CHUNK, s.searcher.getIndexReader().leaves().size()); - for (LeafReaderContext context : s.searcher.getIndexReader().leaves()) { + assertEquals(NUM_DOCS / SEGMENT_CHUNK, s.searcher().getIndexReader().leaves().size()); + for (LeafReaderContext context : s.searcher().getIndexReader().leaves()) { assertEquals(SEGMENT_CHUNK, context.reader().maxDoc()); } } finally { @@ -237,7 +237,7 @@ private void verifyNoTimeout(SearchRequest request) throws Exception { throw new RuntimeException(e); } }); - assertEquals(NUM_DOCS, hits.totalHits.value); + assertEquals(NUM_DOCS, hits.totalHits.value()); assertEquals(NUM_DOCS - 1, hits.scoreDocs[0].score, 0); } @@ -324,7 +324,7 @@ private void verifyPartialResults(SearchRequest request, int expectedHits) throw throw new RuntimeException(e); } }); - assertEquals(expectedHits, hits.totalHits.value); + assertEquals(expectedHits, hits.totalHits.value()); } @Test(expected = CollectionTimeoutException.class) @@ -503,9 +503,9 @@ private TopDocs getTopDocs(SearchContext context, CollectorManager grpcFacetResults = new ArrayList<>(); DrillSideways drillS = new DrillSidewaysImpl( - context.getSearcherAndTaxonomy().searcher, + context.getSearcherAndTaxonomy().searcher(), context.getIndexState().getFacetsConfig(), - context.getSearcherAndTaxonomy().taxonomyReader, + context.getSearcherAndTaxonomy().taxonomyReader(), Collections.singletonList(getTestFacet()), context.getSearcherAndTaxonomy(), context.getIndexState(), @@ -521,7 +521,7 @@ private TopDocs getTopDocs(SearchContext context, CollectorManager mockReferenceManager = mock(ReferenceManager.class); IndexSearcher mockSearcher = mock(IndexSearcher.class); - IndexReader mockIndexReader = mock(IndexReader.class); + IndexReader mockIndexReader = mock(StandardDirectoryReader.class); when(mockSearcher.getIndexReader()).thenReturn(mockIndexReader); when(mockReferenceManager.acquire()).thenReturn(mockSearcher); when(mockPrimaryNode.getSearcherManager()).thenReturn(mockReferenceManager); @@ -76,7 +77,7 @@ public void testRefreshIfNeeded_noopFlush() throws IOException { NRTPrimaryNode mockPrimaryNode = mock(NRTPrimaryNode.class); ReferenceManager mockReferenceManager = mock(ReferenceManager.class); IndexSearcher mockSearcher = mock(IndexSearcher.class); - IndexReader mockIndexReader = mock(IndexReader.class); + IndexReader mockIndexReader = mock(StandardDirectoryReader.class); when(mockSearcher.getIndexReader()).thenReturn(mockIndexReader); when(mockReferenceManager.acquire()).thenReturn(mockSearcher); when(mockPrimaryNode.getSearcherManager()).thenReturn(mockReferenceManager); @@ -104,7 +105,7 @@ public void testRefreshIfNeededWatcher() throws IOException { NRTPrimaryNode mockPrimaryNode = mock(NRTPrimaryNode.class); ReferenceManager mockReferenceManager = mock(ReferenceManager.class); IndexSearcher mockSearcher = mock(IndexSearcher.class); - IndexReader mockIndexReader = mock(IndexReader.class); + IndexReader mockIndexReader = mock(StandardDirectoryReader.class); when(mockSearcher.getIndexReader()).thenReturn(mockIndexReader); when(mockReferenceManager.acquire()).thenReturn(mockSearcher); when(mockPrimaryNode.getSearcherManager()).thenReturn(mockReferenceManager); @@ -149,7 +150,7 @@ public void testRefreshIfNeededWatcher_noopFlush() throws IOException { NRTPrimaryNode mockPrimaryNode = mock(NRTPrimaryNode.class); ReferenceManager mockReferenceManager = mock(ReferenceManager.class); IndexSearcher mockSearcher = mock(IndexSearcher.class); - IndexReader mockIndexReader = mock(IndexReader.class); + IndexReader mockIndexReader = mock(StandardDirectoryReader.class); when(mockSearcher.getIndexReader()).thenReturn(mockIndexReader); when(mockReferenceManager.acquire()).thenReturn(mockSearcher); when(mockPrimaryNode.getSearcherManager()).thenReturn(mockReferenceManager); @@ -193,7 +194,7 @@ public void testRefreshIfNeededWatcher_errorBeforeEnqueue() NRTPrimaryNode mockPrimaryNode = mock(NRTPrimaryNode.class); ReferenceManager mockReferenceManager = mock(ReferenceManager.class); IndexSearcher mockSearcher = mock(IndexSearcher.class); - IndexReader mockIndexReader = mock(IndexReader.class); + IndexReader mockIndexReader = mock(StandardDirectoryReader.class); when(mockSearcher.getIndexReader()).thenReturn(mockIndexReader); when(mockReferenceManager.acquire()).thenReturn(mockSearcher); when(mockPrimaryNode.getSearcherManager()).thenReturn(mockReferenceManager); @@ -234,7 +235,7 @@ public void testRefreshIfNeededWatcher_errorAfterEnqueue() throws IOException { NRTPrimaryNode mockPrimaryNode = mock(NRTPrimaryNode.class); ReferenceManager mockReferenceManager = mock(ReferenceManager.class); IndexSearcher mockSearcher = mock(IndexSearcher.class); - IndexReader mockIndexReader = mock(IndexReader.class); + IndexReader mockIndexReader = mock(StandardDirectoryReader.class); when(mockSearcher.getIndexReader()).thenReturn(mockIndexReader); when(mockReferenceManager.acquire()).thenReturn(mockSearcher); when(mockPrimaryNode.getSearcherManager()).thenReturn(mockReferenceManager); diff --git a/src/test/java/com/yelp/nrtsearch/server/nrt/state/NrtPointStateTest.java b/src/test/java/com/yelp/nrtsearch/server/nrt/state/NrtPointStateTest.java index f6a4554eb..ce1a5ab7c 100644 --- a/src/test/java/com/yelp/nrtsearch/server/nrt/state/NrtPointStateTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/nrt/state/NrtPointStateTest.java @@ -74,16 +74,16 @@ public void testToCopyState() { NrtPointState nrtPointState = new NrtPointState(getCopyState(), Map.of("file3", nrtFileMetaData), primaryId); CopyState copyState = nrtPointState.toCopyState(); - assertEquals(copyState.version, version); - assertEquals(copyState.gen, gen); - assertArrayEquals(copyState.infosBytes, infosBytes); - assertEquals(copyState.primaryGen, primaryGen); - assertEquals(copyState.completedMergeFiles, completedMergeFiles); - assertEquals(copyState.files.size(), 1); - assertEquals(copyState.files.get("file3").length, 10); - assertEquals(copyState.files.get("file3").checksum, 25); - assertArrayEquals(copyState.files.get("file3").footer, new byte[] {0, 10, 11}); - assertArrayEquals(copyState.files.get("file3").header, new byte[] {6, 7, 8}); + assertEquals(copyState.version(), version); + assertEquals(copyState.gen(), gen); + assertArrayEquals(copyState.infosBytes(), infosBytes); + assertEquals(copyState.primaryGen(), primaryGen); + assertEquals(copyState.completedMergeFiles(), completedMergeFiles); + assertEquals(copyState.files().size(), 1); + assertEquals(copyState.files().get("file3").length(), 10); + assertEquals(copyState.files().get("file3").checksum(), 25); + assertArrayEquals(copyState.files().get("file3").footer(), new byte[] {0, 10, 11}); + assertArrayEquals(copyState.files().get("file3").header(), new byte[] {6, 7, 8}); } @Test diff --git a/src/test/java/com/yelp/nrtsearch/server/rescore/RescorerTest.java b/src/test/java/com/yelp/nrtsearch/server/rescore/RescorerTest.java index 875805ec8..b383c8883 100644 --- a/src/test/java/com/yelp/nrtsearch/server/rescore/RescorerTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/rescore/RescorerTest.java @@ -134,7 +134,7 @@ public TopDocs rescore(TopDocs hits, RescoreContext context) throws IOException context .getSearchContext() .getSearcherAndTaxonomy() - .searcher + .searcher() .getIndexReader() .leaves(); int leafIndex = ReaderUtil.subIndex(doc.doc, leaves); diff --git a/src/test/java/com/yelp/nrtsearch/server/search/GlobalOrdinalLookupTest.java b/src/test/java/com/yelp/nrtsearch/server/search/GlobalOrdinalLookupTest.java index a405537f5..7cdc30a04 100644 --- a/src/test/java/com/yelp/nrtsearch/server/search/GlobalOrdinalLookupTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/search/GlobalOrdinalLookupTest.java @@ -71,8 +71,9 @@ public void testEmptyIndex() throws IOException { ShardState shardState = indexState.getShard(0); try { s = shardState.acquire(); - assertEmptyLookup(indexState.getFieldOrThrow(VALUE_FIELD), s.searcher.getIndexReader()); - assertEmptyLookup(indexState.getFieldOrThrow(VALUE_MULTI_FIELD), s.searcher.getIndexReader()); + assertEmptyLookup(indexState.getFieldOrThrow(VALUE_FIELD), s.searcher().getIndexReader()); + assertEmptyLookup( + indexState.getFieldOrThrow(VALUE_MULTI_FIELD), s.searcher().getIndexReader()); } finally { if (s != null) { shardState.release(s); @@ -103,9 +104,9 @@ public void testSingleSegment() throws Exception { ShardState shardState = indexState.getShard(0); try { s = shardState.acquire(); - assertSingleLookup(indexState.getFieldOrThrow(VALUE_FIELD), s.searcher.getIndexReader()); + assertSingleLookup(indexState.getFieldOrThrow(VALUE_FIELD), s.searcher().getIndexReader()); assertSingleLookup( - indexState.getFieldOrThrow(VALUE_MULTI_FIELD), s.searcher.getIndexReader()); + indexState.getFieldOrThrow(VALUE_MULTI_FIELD), s.searcher().getIndexReader()); } finally { if (s != null) { shardState.release(s); @@ -132,8 +133,9 @@ public void testMultiSegment() throws Exception { ShardState shardState = indexState.getShard(0); try { s = shardState.acquire(); - assertMultiLookup(indexState.getFieldOrThrow(VALUE_FIELD), s.searcher.getIndexReader()); - assertMultiLookup(indexState.getFieldOrThrow(VALUE_MULTI_FIELD), s.searcher.getIndexReader()); + assertMultiLookup(indexState.getFieldOrThrow(VALUE_FIELD), s.searcher().getIndexReader()); + assertMultiLookup( + indexState.getFieldOrThrow(VALUE_MULTI_FIELD), s.searcher().getIndexReader()); } finally { if (s != null) { shardState.release(s); diff --git a/src/test/java/com/yelp/nrtsearch/server/search/MyIndexSearcherTest.java b/src/test/java/com/yelp/nrtsearch/server/search/MyIndexSearcherTest.java index a086d431d..b99c92bb0 100644 --- a/src/test/java/com/yelp/nrtsearch/server/search/MyIndexSearcherTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/search/MyIndexSearcherTest.java @@ -16,7 +16,6 @@ package com.yelp.nrtsearch.server.search; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import com.yelp.nrtsearch.server.ServerTestCase; @@ -25,7 +24,6 @@ import com.yelp.nrtsearch.server.grpc.LiveSettingsRequest; import com.yelp.nrtsearch.server.index.IndexState; import com.yelp.nrtsearch.server.index.ShardState; -import com.yelp.nrtsearch.server.search.MyIndexSearcher.ExecutorWithParams; import io.grpc.testing.GrpcCleanupRule; import java.io.IOException; import java.util.ArrayList; @@ -136,14 +134,11 @@ private void assertSliceParams(String index, int maxDocs, int maxSegments) throw ShardState shardState = indexState.getShard(0); try { s = shardState.acquire(); - assertTrue(s.searcher instanceof MyIndexSearcher); - MyIndexSearcher searcher = (MyIndexSearcher) s.searcher; - assertTrue(searcher.getExecutor() instanceof MyIndexSearcher.ExecutorWithParams); - MyIndexSearcher.ExecutorWithParams params = - (MyIndexSearcher.ExecutorWithParams) searcher.getExecutor(); - assertSame(indexState.getSearchExecutor(), params.wrapped); - assertEquals(maxDocs, params.sliceMaxDocs); - assertEquals(maxSegments, params.sliceMaxSegments); + assertTrue(s.searcher() instanceof MyIndexSearcher); + MyIndexSearcher searcher = (MyIndexSearcher) s.searcher(); + MyIndexSearcher.SlicingParams params = searcher.getSlicingParams(); + assertEquals(maxDocs, params.sliceMaxDocs()); + assertEquals(maxSegments, params.sliceMaxSegments()); } finally { if (s != null) { shardState.release(s); @@ -157,12 +152,12 @@ public void testSliceDocsLimit() throws IOException { ShardState shardState = getGlobalState().getIndexOrThrow(DOCS_INDEX).getShard(0); try { s = shardState.acquire(); - LeafSlice[] slices = s.searcher.getSlices(); + LeafSlice[] slices = s.searcher().getSlices(); assertEquals(4, slices.length); - assertEquals(3, slices[0].leaves.length); - assertEquals(3, slices[1].leaves.length); - assertEquals(3, slices[2].leaves.length); - assertEquals(1, slices[3].leaves.length); + assertEquals(3, slices[0].partitions.length); + assertEquals(3, slices[1].partitions.length); + assertEquals(3, slices[2].partitions.length); + assertEquals(1, slices[3].partitions.length); } finally { if (s != null) { shardState.release(s); @@ -176,20 +171,15 @@ public void testSliceSegmentsLimit() throws IOException { ShardState shardState = getGlobalState().getIndexOrThrow(SEGMENTS_INDEX).getShard(0); try { s = shardState.acquire(); - LeafSlice[] slices = s.searcher.getSlices(); + LeafSlice[] slices = s.searcher().getSlices(); assertEquals(3, slices.length); - assertEquals(4, slices[0].leaves.length); - assertEquals(4, slices[1].leaves.length); - assertEquals(2, slices[2].leaves.length); + assertEquals(4, slices[0].partitions.length); + assertEquals(4, slices[1].partitions.length); + assertEquals(2, slices[2].partitions.length); } finally { if (s != null) { shardState.release(s); } } } - - @Test(expected = NullPointerException.class) - public void testNullWrappedExecutor() throws IOException { - new ExecutorWithParams(null, 10, 10, 1); - } } diff --git a/src/test/java/com/yelp/nrtsearch/server/search/MyIndexSearcherVirtualShardsTest.java b/src/test/java/com/yelp/nrtsearch/server/search/MyIndexSearcherVirtualShardsTest.java index 1eb8edd9c..7df3a2e59 100644 --- a/src/test/java/com/yelp/nrtsearch/server/search/MyIndexSearcherVirtualShardsTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/search/MyIndexSearcherVirtualShardsTest.java @@ -115,12 +115,10 @@ public void testHasVirtualShards() throws Exception { ShardState shardState = getGlobalState().getIndexOrThrow(DEFAULT_TEST_INDEX).getShard(0); try { s = shardState.acquire(); - assertTrue(s.searcher instanceof MyIndexSearcher); - MyIndexSearcher searcher = (MyIndexSearcher) s.searcher; - assertTrue(searcher.getExecutor() instanceof MyIndexSearcher.ExecutorWithParams); - MyIndexSearcher.ExecutorWithParams params = - (MyIndexSearcher.ExecutorWithParams) searcher.getExecutor(); - assertEquals(111, params.virtualShards); + assertTrue(s.searcher() instanceof MyIndexSearcher); + MyIndexSearcher searcher = (MyIndexSearcher) s.searcher(); + MyIndexSearcher.SlicingParams params = searcher.getSlicingParams(); + assertEquals(111, params.virtualShards()); } finally { if (s != null) { shardState.release(s); @@ -182,16 +180,11 @@ private void assertSlices(List docCounts, List segmentCounts) ShardState shardState = getGlobalState().getIndexOrThrow(DEFAULT_TEST_INDEX).getShard(0); try { s = shardState.acquire(); - LeafSlice[] slices = s.searcher.getSlices(); + LeafSlice[] slices = s.searcher().getSlices(); assertEquals(docCounts.size(), slices.length); for (int i = 0; i < docCounts.size(); ++i) { - assertEquals(segmentCounts.get(i), Integer.valueOf(slices[i].leaves.length)); - - int totalDocs = 0; - for (int j = 0; j < slices[i].leaves.length; ++j) { - totalDocs += slices[i].leaves[j].reader().numDocs(); - } - assertEquals(docCounts.get(i), Integer.valueOf(totalDocs)); + assertEquals(segmentCounts.get(i), Integer.valueOf(slices[i].partitions.length)); + assertEquals(docCounts.get(i), Integer.valueOf(slices[i].getMaxDocs())); } } finally { if (s != null) { diff --git a/src/test/java/com/yelp/nrtsearch/server/search/collectors/MyTopSuggestDocsCollectorTest.java b/src/test/java/com/yelp/nrtsearch/server/search/collectors/MyTopSuggestDocsCollectorTest.java index e86e610c6..c8bd1a486 100644 --- a/src/test/java/com/yelp/nrtsearch/server/search/collectors/MyTopSuggestDocsCollectorTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/search/collectors/MyTopSuggestDocsCollectorTest.java @@ -81,8 +81,8 @@ public void shouldGetTopScoringSuggestScoreDocsUniqueById() throws IOException { TopSuggestDocs reduced = manager.reduce(collectors); // check total hits - assertEquals(5, reduced.totalHits.value); - assertEquals(TotalHits.Relation.EQUAL_TO, reduced.totalHits.relation); + assertEquals(5, reduced.totalHits.value()); + assertEquals(TotalHits.Relation.EQUAL_TO, reduced.totalHits.relation()); // check suggest score docs assertEquals(expectedFinalSuggestScoreDocs.length, reduced.scoreDocs.length); @@ -122,8 +122,8 @@ public void shouldGetTopScoringSuggestScoreDocsUniqueById() throws IOException { TopSuggestDocs reduced = manager.reduce(collectors); // check total hits - assertEquals(2, reduced.totalHits.value); - assertEquals(TotalHits.Relation.EQUAL_TO, reduced.totalHits.relation); + assertEquals(2, reduced.totalHits.value()); + assertEquals(TotalHits.Relation.EQUAL_TO, reduced.totalHits.relation()); // check suggest score docs assertEquals(expectedFinalSuggestScoreDocs.length, reduced.scoreDocs.length); diff --git a/src/test/java/com/yelp/nrtsearch/server/state/BackendGlobalStateTest.java b/src/test/java/com/yelp/nrtsearch/server/state/BackendGlobalStateTest.java index e5eeb019c..cd36c3e7f 100644 --- a/src/test/java/com/yelp/nrtsearch/server/state/BackendGlobalStateTest.java +++ b/src/test/java/com/yelp/nrtsearch/server/state/BackendGlobalStateTest.java @@ -63,6 +63,7 @@ import java.util.Set; import org.apache.lucene.facet.taxonomy.SearcherTaxonomyManager.SearcherAndTaxonomy; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.StandardDirectoryReader; import org.apache.lucene.search.IndexSearcher; import org.junit.Before; import org.junit.BeforeClass; @@ -1005,7 +1006,7 @@ public void testStartIndex() throws IOException { mockManagers.put(BackendGlobalState.getUniqueIndexName("test_index", "test_id_1"), mockManager); ShardState mockShardState = mock(ShardState.class); - IndexReader mockReader = mock(IndexReader.class); + IndexReader mockReader = mock(StandardDirectoryReader.class); IndexSearcher mockSearcher = mock(IndexSearcher.class); when(mockState.getShard(0)).thenReturn(mockShardState); when(mockSearcher.getIndexReader()).thenReturn(mockReader); @@ -1067,7 +1068,7 @@ public void testStartIndexPrimary() throws IOException { mockManagers.put(BackendGlobalState.getUniqueIndexName("test_index", "test_id_1"), mockManager); ShardState mockShardState = mock(ShardState.class); - IndexReader mockReader = mock(IndexReader.class); + IndexReader mockReader = mock(StandardDirectoryReader.class); IndexSearcher mockSearcher = mock(IndexSearcher.class); when(mockState.getShard(0)).thenReturn(mockShardState); when(mockSearcher.getIndexReader()).thenReturn(mockReader); @@ -1134,7 +1135,7 @@ public void testStartIndexReplica() throws IOException { mockManagers.put(BackendGlobalState.getUniqueIndexName("test_index", "test_id_1"), mockManager); ShardState mockShardState = mock(ShardState.class); - IndexReader mockReader = mock(IndexReader.class); + IndexReader mockReader = mock(StandardDirectoryReader.class); IndexSearcher mockSearcher = mock(IndexSearcher.class); when(mockState.getShard(0)).thenReturn(mockShardState); when(mockSearcher.getIndexReader()).thenReturn(mockReader);