diff --git a/CHANGELOG.md b/CHANGELOG.md index aa57a37ffdd38..6abf3a04b2475 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 3.x] ### Added +- Add Roaring64NavigableMap support for bitmap filtering on long fields ([#20598](https://github.com/opensearch-project/OpenSearch/pull/20598)) - Add getWrappedScorer method to ProfileScorer for plugin access to wrapped scorers ([#20548](https://github.com/opensearch-project/OpenSearch/issues/20548)) - Support expected cluster name with validation in CCS Sniff mode ([#20532](https://github.com/opensearch-project/OpenSearch/pull/20532)) - Add security policy to allow `accessUnixDomainSocket` in `transport-grpc` module ([#20463](https://github.com/opensearch-project/OpenSearch/pull/20463)) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/380_bitmap_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/380_bitmap_filtering.yml index c885e3fbc2446..8fd71cb5b70ee 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/380_bitmap_filtering.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/380_bitmap_filtering.yml @@ -182,3 +182,114 @@ setup: - match: { hits.hits.1._source.student_id: 111 } - match: { hits.hits.2._source.name: John Doe } - match: { hits.hits.2._source.student_id: 333 } + +--- +"Terms query accepting bitmap for long field": + - skip: + version: " - 3.5.99" + reason: The bitmap filtering for long fields is available in 3.6 and later. + - do: + indices.create: + index: students_long + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + student_id: + type: long + - do: + bulk: + refresh: true + body: + - { "index": { "_index": "students_long", "_id": "1" } } + - { "name": "Jane Doe", "student_id": 111 } + - { "index": { "_index": "students_long", "_id": "2" } } + - { "name": "Mary Major", "student_id": 222 } + - { "index": { "_index": "students_long", "_id": "3" } } + - { "name": "John Doe", "student_id": 333 } + - do: + search: + rest_total_hits_as_int: true + index: students_long + body: { + "query": { + "terms": { + "student_id": ["AQAAAAAAAAAAAAAAOjAAAAEAAAAAAAEAEAAAAG8A3gA="], + "value_type": "bitmap" + } + } + } + - match: { hits.total: 2 } + - match: { hits.hits.0._source.name: Jane Doe } + - match: { hits.hits.0._source.student_id: 111 } + - match: { hits.hits.1._source.name: Mary Major } + - match: { hits.hits.1._source.student_id: 222 } + +--- +"Terms lookup on a binary field with bitmap for long field": + - skip: + version: " - 3.5.99" + reason: The bitmap filtering for long fields is available in 3.6 and later. + - do: + indices.create: + index: students_long_lookup + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + student_id: + type: long + - do: + bulk: + refresh: true + body: + - { "index": { "_index": "students_long_lookup", "_id": "1" } } + - { "name": "Jane Doe", "student_id": 111 } + - { "index": { "_index": "students_long_lookup", "_id": "2" } } + - { "name": "Mary Major", "student_id": 222 } + - { "index": { "_index": "students_long_lookup", "_id": "3" } } + - { "name": "John Doe", "student_id": 333 } + - do: + indices.create: + index: classes_long + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + enrolled: + type: binary + store: true + - do: + bulk: + refresh: true + body: + - { "index": { "_index": "classes_long", "_id": "101" } } + - { "enrolled": "AQAAAAAAAAAAAAAAOjAAAAEAAAAAAAEAEAAAAG8A3gA=" } + - do: + search: + rest_total_hits_as_int: true + index: students_long_lookup + body: { + "query": { + "terms": { + "student_id": { + "index": "classes_long", + "id": "101", + "path": "enrolled", + "store": true + }, + "value_type": "bitmap" + } + } + } + - match: { hits.total: 2 } + - match: { hits.hits.0._source.name: Jane Doe } + - match: { hits.hits.0._source.student_id: 111 } + - match: { hits.hits.1._source.name: Mary Major } + - match: { hits.hits.1._source.student_id: 222 } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java b/server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java index cf4b0106f8090..34293146d73cc 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java @@ -84,6 +84,8 @@ import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase; import org.opensearch.test.junit.annotations.TestIssueLogging; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; import java.io.IOException; import java.io.Reader; import java.nio.ByteBuffer; @@ -102,6 +104,7 @@ import java.util.regex.Pattern; import org.roaringbitmap.RoaringBitmap; +import org.roaringbitmap.longlong.Roaring64NavigableMap; import static java.util.Collections.singletonMap; import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; @@ -1197,6 +1200,74 @@ public void testTermsQueryWithBitmapDocValuesQuery() throws Exception { assertSearchHits(searchResponse, "1", "3", "4"); } + public void testTermsQueryWithBitmapLongField() throws Exception { + assertAcked( + prepareCreate("products_long").setMapping( + jsonBuilder().startObject() + .startObject("properties") + .startObject("product") + .field("type", "long") + .endObject() + .endObject() + .endObject() + ) + ); + indexRandom( + true, + client().prepareIndex("products_long").setId("1").setSource("product", 1L), + client().prepareIndex("products_long").setId("2").setSource("product", 2L), + client().prepareIndex("products_long").setId("3").setSource("product", new long[] { 1L, 3L }), + client().prepareIndex("products_long").setId("4").setSource("product", 4L) + ); + + Roaring64NavigableMap r = new Roaring64NavigableMap(true); + r.addLong(1L); + r.addLong(4L); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + r.serializePortable(new DataOutputStream(baos)); + BytesArray bitmap = new BytesArray(baos.toByteArray()); + SearchResponse searchResponse = client().prepareSearch("products_long") + .setQuery(constantScoreQuery(termsQuery("product", bitmap).valueType(TermsQueryBuilder.ValueType.BITMAP))) + .get(); + assertHitCount(searchResponse, 3L); + assertSearchHits(searchResponse, "1", "3", "4"); + } + + public void testTermsQueryWithBitmapLongFieldLargeValues() throws Exception { + assertAcked( + prepareCreate("products_long_large").setMapping( + jsonBuilder().startObject() + .startObject("properties") + .startObject("product") + .field("type", "long") + .endObject() + .endObject() + .endObject() + ) + ); + long largeVal1 = Integer.MAX_VALUE + 100L; + long largeVal2 = Integer.MAX_VALUE + 200L; + long largeVal3 = Integer.MAX_VALUE + 300L; + indexRandom( + true, + client().prepareIndex("products_long_large").setId("1").setSource("product", largeVal1), + client().prepareIndex("products_long_large").setId("2").setSource("product", largeVal2), + client().prepareIndex("products_long_large").setId("3").setSource("product", largeVal3) + ); + + Roaring64NavigableMap r = new Roaring64NavigableMap(true); + r.addLong(largeVal1); + r.addLong(largeVal3); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + r.serializePortable(new DataOutputStream(baos)); + BytesArray bitmap = new BytesArray(baos.toByteArray()); + SearchResponse searchResponse = client().prepareSearch("products_long_large") + .setQuery(constantScoreQuery(termsQuery("product", bitmap).valueType(TermsQueryBuilder.ValueType.BITMAP))) + .get(); + assertHitCount(searchResponse, 2L); + assertSearchHits(searchResponse, "1", "3"); + } + public void testTermsLookupFilter() throws Exception { assertAcked(prepareCreate("lookup").setMapping("terms", "type=text", "other", "type=text")); indexRandomForConcurrentSearch("lookup"); diff --git a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java index 321ae22158bc9..4c836501f5d87 100644 --- a/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java @@ -74,9 +74,13 @@ import org.opensearch.search.approximate.ApproximatePointRangeQuery; import org.opensearch.search.approximate.ApproximateScoreQuery; import org.opensearch.search.lookup.SearchLookup; +import org.opensearch.search.query.Bitmap64DocValuesQuery; +import org.opensearch.search.query.Bitmap64IndexQuery; import org.opensearch.search.query.BitmapDocValuesQuery; import org.opensearch.search.query.BitmapIndexQuery; +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; import java.io.IOException; import java.math.BigInteger; import java.nio.ByteBuffer; @@ -93,6 +97,8 @@ import java.util.function.Supplier; import org.roaringbitmap.RoaringBitmap; +import org.roaringbitmap.longlong.LongIterator; +import org.roaringbitmap.longlong.Roaring64NavigableMap; /** * A {@link FieldMapper} for numeric types: byte, short, int, long, float, double and unsigned long. @@ -1162,7 +1168,27 @@ public Query bitmapQuery(String field, BytesArray bitmapArray, boolean isSearcha try { bitmap.deserialize(ByteBuffer.wrap(bitmapArray.array())); } catch (Exception e) { - throw new IllegalArgumentException("Failed to deserialize the bitmap.", e); + // Fallback: try 64-bit Roaring64NavigableMap and down-convert. + // The two formats have distinct cookies so deserialization failure is reliable. + // All values must fit in [Integer.MIN_VALUE, Integer.MAX_VALUE] or an error is thrown. + try { + Roaring64NavigableMap bitmap64 = new Roaring64NavigableMap(true); + bitmap64.deserializePortable(new DataInputStream(new ByteArrayInputStream(bitmapArray.array()))); + LongIterator iter = bitmap64.getLongIterator(); + while (iter.hasNext()) { + long value = iter.next(); + if (value < Integer.MIN_VALUE || value > Integer.MAX_VALUE) { + throw new IllegalArgumentException( + "Bitmap contains value " + value + " which is out of range for integer field" + ); + } + bitmap.add((int) value); + } + } catch (IllegalArgumentException iae) { + throw iae; + } catch (Exception e2) { + throw new IllegalArgumentException("Failed to deserialize the bitmap.", e); + } } if (isSearchable && hasDocValues) { @@ -1437,11 +1463,59 @@ public List createFields( return fields; } + /** + * Bitmap query support for long fields using Roaring64NavigableMap with portable serialization. + *

+ * Signed mode (signedLongs=true) is required so the bitmap iterator produces values in the + * same order as Lucene's LongPoint BKD tree encoding. The default Roaring64NavigableMap + * constructor uses unsigned mode, which would break the merge-join for negative values. + *

+ * Clients should serialize bitmaps using {@code Roaring64NavigableMap.serializePortable()}. + * If a 32-bit RoaringBitmap blob is received (detected via cookie-based format validation), + * values are up-converted from int to long, which is always safe. + *

+ * Cross-language compatibility: other implementations (C/CRoaring, Go, Python) typically + * use unsigned 64-bit semantics. For values in the range 0 to 2^63-1, the bit patterns + * are identical and fully interoperable. Negative Java longs correspond to unsigned values + * greater than or equal to 2^63 in other implementations. + *

+ * Not applicable to unsigned_long fields, which use BigIntegerPoint (16-byte encoding) + * and are incompatible with the 8-byte Roaring64NavigableMap representation. + */ + @Override + public Query bitmapQuery(String field, BytesArray bitmapArray, boolean isSearchable, boolean hasDocValues) { + // signedLongs=true is critical: ensures iterator order matches LongPoint BKD tree sort order + Roaring64NavigableMap bitmap = new Roaring64NavigableMap(true); + try { + bitmap.deserializePortable(new DataInputStream(new ByteArrayInputStream(bitmapArray.array()))); + } catch (Exception e) { + // Fallback: try 32-bit RoaringBitmap and up-convert (int -> long is always safe). + // The two formats have distinct cookies so deserialization failure is reliable. + try { + RoaringBitmap bitmap32 = new RoaringBitmap(); + bitmap32.deserialize(ByteBuffer.wrap(bitmapArray.array())); + bitmap32.forEach((int value) -> bitmap.addLong(value)); + } catch (Exception e2) { + throw new IllegalArgumentException("Failed to deserialize the bitmap.", e); + } + } + + if (isSearchable && hasDocValues) { + return new IndexOrDocValuesQuery(new Bitmap64IndexQuery(field, bitmap), new Bitmap64DocValuesQuery(field, bitmap)); + } + if (isSearchable) { + return new Bitmap64IndexQuery(field, bitmap); + } + return new Bitmap64DocValuesQuery(field, bitmap); + } + @Override Number valueForSearch(String value) { return Long.parseLong(value); } }, + // Note: UNSIGNED_LONG does not support bitmap queries. It uses BigIntegerPoint (16-byte + // encoding) which is incompatible with Roaring64NavigableMap's 8-byte long representation. UNSIGNED_LONG("unsigned_long", NumericType.UNSIGNED_LONG) { @Override public BigInteger parse(Object value, boolean coerce) { diff --git a/server/src/main/java/org/opensearch/search/query/Bitmap64DocValuesQuery.java b/server/src/main/java/org/opensearch/search/query/Bitmap64DocValuesQuery.java new file mode 100644 index 0000000000000..b5c5c6c91f201 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/query/Bitmap64DocValuesQuery.java @@ -0,0 +1,161 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query; + +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.RamUsageEstimator; + +import java.io.IOException; +import java.util.Objects; + +import org.roaringbitmap.longlong.Roaring64NavigableMap; + +import static org.opensearch.search.query.Bitmap64IndexQuery.checkArgs; + +/** + * Filter with 64-bit bitmap for long fields using doc values. + *

+ * Similar to Lucene SortedNumericDocValuesSetQuery but for 64-bit values. + * The bitmap must be constructed with {@code new Roaring64NavigableMap(true)} (signed mode) + * so that {@code first()} and {@code last()} return correct signed min/max bounds for the + * range optimization in the two-phase iterator. + * + * @see Bitmap64IndexQuery + */ +public class Bitmap64DocValuesQuery extends Query implements Accountable { + + final String field; + final Roaring64NavigableMap bitmap; + final long min; + final long max; + + public Bitmap64DocValuesQuery(String field, Roaring64NavigableMap bitmap) { + checkArgs(field, bitmap); + this.field = field; + this.bitmap = bitmap; + if (!bitmap.isEmpty()) { + min = bitmap.first(); + max = bitmap.last(); + } else { + min = 0; + max = 0; + } + } + + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { + @Override + public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { + SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field); + final NumericDocValues singleton = DocValues.unwrapSingleton(values); + final TwoPhaseIterator iterator; + if (singleton != null) { + iterator = new TwoPhaseIterator(singleton) { + @Override + public boolean matches() throws IOException { + long value = singleton.longValue(); + return value >= min && value <= max && bitmap.contains(value); + } + + @Override + public float matchCost() { + return 5; + } + }; + } else { + iterator = new TwoPhaseIterator(values) { + @Override + public boolean matches() throws IOException { + int count = values.docValueCount(); + for (int i = 0; i < count; i++) { + final long value = values.nextValue(); + if (value < min) { + continue; + } else if (value > max) { + return false; + } else if (bitmap.contains(value)) { + return true; + } + } + return false; + } + + @Override + public float matchCost() { + return 5; + } + }; + } + final Scorer scorer = new ConstantScoreScorer(score(), scoreMode, iterator); + return new DefaultScorerSupplier(scorer); + } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return DocValues.isCacheable(ctx, field); + } + }; + } + + @Override + public String toString(String field) { + return "Bitmap64DocValuesQuery(field=" + this.field + ")"; + } + + @Override + public Query rewrite(IndexSearcher indexSearcher) throws IOException { + if (bitmap.isEmpty()) { + return new MatchNoDocsQuery(); + } + return super.rewrite(indexSearcher); + } + + @Override + public boolean equals(Object other) { + if (sameClassAs(other) == false) { + return false; + } + Bitmap64DocValuesQuery that = (Bitmap64DocValuesQuery) other; + return field.equals(that.field) && bitmap.equals(that.bitmap); + } + + @Override + public int hashCode() { + return Objects.hash(classHash(), field, bitmap); + } + + @Override + public long ramBytesUsed() { + return RamUsageEstimator.shallowSizeOfInstance(Bitmap64DocValuesQuery.class) + RamUsageEstimator.sizeOf(field) + bitmap + .getLongSizeInBytes(); + } + + @Override + public void visit(QueryVisitor visitor) { + if (visitor.acceptField(field)) { + visitor.visitLeaf(this); + } + } +} diff --git a/server/src/main/java/org/opensearch/search/query/Bitmap64IndexQuery.java b/server/src/main/java/org/opensearch/search/query/Bitmap64IndexQuery.java new file mode 100644 index 0000000000000..c3d3f89f18a8c --- /dev/null +++ b/server/src/main/java/org/opensearch/search/query/Bitmap64IndexQuery.java @@ -0,0 +1,298 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query; + +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefIterator; +import org.apache.lucene.util.DocIdSetBuilder; +import org.apache.lucene.util.RamUsageEstimator; + +import java.io.IOException; +import java.util.Objects; + +import org.roaringbitmap.longlong.LongIterator; +import org.roaringbitmap.longlong.Roaring64NavigableMap; + +/** + * A query that matches all documents that contain a set of long numbers represented by a 64-bit bitmap. + *

+ * Uses {@link Roaring64NavigableMap} with signed long mode ({@code signedLongs=true}) to match the + * sort order of Lucene's {@link LongPoint} encoding in the BKD tree. The bitmap must be + * constructed with {@code new Roaring64NavigableMap(true)} — unsigned mode will produce incorrect + * results for negative values because the merge-join requires the iterator and point index to + * traverse values in the same order. + *

+ * Supports the full signed {@code long} range ({@code Long.MIN_VALUE} to {@code Long.MAX_VALUE}). + * Not compatible with {@code unsigned_long} fields, which use {@code BigIntegerPoint} (16-byte encoding). + *

+ * Cross-language note: other roaring bitmap implementations (C, Go, Python) typically use unsigned + * 64-bit semantics. For positive values (0 to 2^63-1), the bit patterns are identical and fully + * interoperable. Values outside this range will have different signed/unsigned interpretations + * but the underlying bit patterns remain consistent. + *

+ * Serialization uses the portable format via {@code serializePortable}/{@code deserializePortable}, + * which is structurally compatible with the RoaringFormatSpec 64-bit extension. The cookie-based + * header validation in the roaring bitmap library reliably distinguishes 32-bit and 64-bit formats, + * enabling safe fallback deserialization. + * + * @opensearch.internal + */ +public class Bitmap64IndexQuery extends Query implements Accountable { + + private final Roaring64NavigableMap bitmap; + private final String field; + + public Bitmap64IndexQuery(String field, Roaring64NavigableMap bitmap) { + checkArgs(field, bitmap); + this.bitmap = bitmap; + this.field = field; + } + + static void checkArgs(String field, Roaring64NavigableMap bitmap) { + if (field == null) { + throw new IllegalArgumentException("field must not be null"); + } + if (bitmap == null) { + throw new IllegalArgumentException("bitmap must not be null"); + } + } + + interface Bitmap64Iterator extends BytesRefIterator { + // wrap LongIterator.next() + BytesRef next(); + + // advance as long as the next value is smaller than target + void advance(byte[] target); + } + + private static Bitmap64Iterator bitmap64EncodedIterator(Roaring64NavigableMap bitmap) { + return new Bitmap64Iterator() { + private final LongIterator iterator = bitmap.getLongIterator(); + private final BytesRef encoded = new BytesRef(new byte[Long.BYTES]); + private long buffered; + private boolean hasBuffered; + + { + if (iterator.hasNext()) { + buffered = iterator.next(); + hasBuffered = true; + } + } + + public BytesRef next() { + if (!hasBuffered) { + return null; + } + LongPoint.encodeDimension(buffered, encoded.bytes, 0); + if (iterator.hasNext()) { + buffered = iterator.next(); + } else { + hasBuffered = false; + } + return encoded; + } + + public void advance(byte[] target) { + long targetVal = LongPoint.decodeDimension(target, 0); + while (hasBuffered && buffered < targetVal) { + if (iterator.hasNext()) { + buffered = iterator.next(); + } else { + hasBuffered = false; + } + } + } + }; + } + + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { + final long cardinality = bitmap.getLongCardinality(); + + @Override + public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException { + LeafReader reader = context.reader(); + PointValues values = reader.getPointValues(field); + if (values == null) { + return null; + } + if (values.getNumIndexDimensions() != 1) { + throw new IllegalArgumentException("field must have only one dimension"); + } + + return new ScorerSupplier() { + long cost = -1; + + final DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values); + final MergePointVisitor visitor = new MergePointVisitor(result); + + @Override + public Scorer get(long leadCost) throws IOException { + values.intersect(visitor); + return new ConstantScoreScorer(score(), scoreMode, result.build().iterator()); + } + + @Override + public long cost() { + if (cost == -1) { + cost = cardinality * 20; + } + return cost; + } + }; + } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return true; + } + }; + } + + private class MergePointVisitor implements PointValues.IntersectVisitor { + private final DocIdSetBuilder result; + private final Bitmap64Iterator iterator; + private BytesRef nextQueryPoint; + private final ArrayUtil.ByteArrayComparator comparator; + private DocIdSetBuilder.BulkAdder adder; + + public MergePointVisitor(DocIdSetBuilder result) throws IOException { + this.result = result; + this.comparator = ArrayUtil.getUnsignedComparator(Long.BYTES); + this.iterator = bitmap64EncodedIterator(bitmap); + nextQueryPoint = iterator.next(); + } + + @Override + public void grow(int count) { + adder = result.grow(count); + } + + @Override + public void visit(int docID) { + adder.add(docID); + } + + @Override + public void visit(DocIdSetIterator iterator) throws IOException { + adder.add(iterator); + } + + @Override + public void visit(int docID, byte[] packedValue) { + if (matches(packedValue)) { + visit(docID); + } + } + + @Override + public void visit(DocIdSetIterator iterator, byte[] packedValue) throws IOException { + if (matches(packedValue)) { + adder.add(iterator); + } + } + + private boolean matches(byte[] packedValue) { + while (nextQueryPoint != null) { + int cmp = comparator.compare(nextQueryPoint.bytes, nextQueryPoint.offset, packedValue, 0); + if (cmp == 0) { + return true; + } else if (cmp < 0) { + iterator.advance(packedValue); + nextQueryPoint = iterator.next(); + } else { + break; + } + } + return false; + } + + @Override + public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + while (nextQueryPoint != null) { + int cmpMin = comparator.compare(nextQueryPoint.bytes, nextQueryPoint.offset, minPackedValue, 0); + if (cmpMin < 0) { + iterator.advance(minPackedValue); + nextQueryPoint = iterator.next(); + continue; + } + int cmpMax = comparator.compare(nextQueryPoint.bytes, nextQueryPoint.offset, maxPackedValue, 0); + if (cmpMax > 0) { + return PointValues.Relation.CELL_OUTSIDE_QUERY; + } + + if (cmpMin == 0 && cmpMax == 0) { + return PointValues.Relation.CELL_INSIDE_QUERY; + } else { + return PointValues.Relation.CELL_CROSSES_QUERY; + } + } + + return PointValues.Relation.CELL_OUTSIDE_QUERY; + } + } + + @Override + public Query rewrite(IndexSearcher indexSearcher) throws IOException { + if (bitmap.isEmpty()) { + return new MatchNoDocsQuery(); + } + return super.rewrite(indexSearcher); + } + + @Override + public String toString(String field) { + return "Bitmap64IndexQuery(field=" + this.field + ")"; + } + + @Override + public void visit(QueryVisitor visitor) { + if (visitor.acceptField(field)) { + visitor.visitLeaf(this); + } + } + + @Override + public boolean equals(Object other) { + if (sameClassAs(other) == false) { + return false; + } + Bitmap64IndexQuery that = (Bitmap64IndexQuery) other; + return field.equals(that.field) && bitmap.equals(that.bitmap); + } + + @Override + public int hashCode() { + return Objects.hash(classHash(), field, bitmap); + } + + @Override + public long ramBytesUsed() { + return RamUsageEstimator.shallowSizeOfInstance(Bitmap64IndexQuery.class) + RamUsageEstimator.sizeOf(field) + bitmap + .getLongSizeInBytes(); + } +} diff --git a/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java index 5f2ba1f083cc5..26b7d0d10afd9 100644 --- a/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/NumberFieldTypeTests.java @@ -77,11 +77,15 @@ import org.opensearch.search.MultiValueMode; import org.opensearch.search.approximate.ApproximatePointRangeQuery; import org.opensearch.search.approximate.ApproximateScoreQuery; +import org.opensearch.search.query.Bitmap64DocValuesQuery; +import org.opensearch.search.query.Bitmap64IndexQuery; import org.opensearch.search.query.BitmapDocValuesQuery; import org.opensearch.search.query.BitmapIndexQuery; import org.junit.Before; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; @@ -93,6 +97,7 @@ import java.util.function.Supplier; import org.roaringbitmap.RoaringBitmap; +import org.roaringbitmap.longlong.Roaring64NavigableMap; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.either; @@ -1018,12 +1023,42 @@ public void testBitmapQuery() throws IOException { w.close(); dir.close(); - NumberType type = randomValueOtherThan(NumberType.INTEGER, () -> randomFrom(NumberType.values())); + NumberType type = randomValueOtherThanMany( + t -> t == NumberType.INTEGER || t == NumberType.LONG, + () -> randomFrom(NumberType.values()) + ); ft = new NumberFieldMapper.NumberFieldType("field", type); NumberFieldType finalFt = ft; assertThrows(IllegalArgumentException.class, () -> finalFt.bitmapQuery(bitmap)); } + public void testBitmapQueryLong() throws IOException { + Roaring64NavigableMap r = new Roaring64NavigableMap(true); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + r.serializePortable(new DataOutputStream(baos)); + BytesArray bitmap = new BytesArray(baos.toByteArray()); + + NumberFieldType ft = new NumberFieldMapper.NumberFieldType("field", NumberType.LONG); + assertEquals( + new IndexOrDocValuesQuery(new Bitmap64IndexQuery("field", r), new Bitmap64DocValuesQuery("field", r)), + ft.bitmapQuery(bitmap) + ); + + ft = new NumberFieldType("field", NumberType.LONG, false, false, true, true, true, null, Collections.emptyMap()); + assertEquals(new Bitmap64DocValuesQuery("field", r), ft.bitmapQuery(bitmap)); + + ft = new NumberFieldType("field", NumberType.LONG, true, false, false, false, true, null, Collections.emptyMap()); + assertEquals(new Bitmap64IndexQuery("field", r), ft.bitmapQuery(bitmap)); + + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig()); + DirectoryReader reader = DirectoryReader.open(w); + assertEquals(new MatchNoDocsQuery(), ft.bitmapQuery(bitmap).rewrite(newSearcher(reader))); + reader.close(); + w.close(); + dir.close(); + } + public void testFetchUnsignedLongDocValues() throws IOException { Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null)); diff --git a/server/src/test/java/org/opensearch/search/query/Bitmap64DocValuesQueryTests.java b/server/src/test/java/org/opensearch/search/query/Bitmap64DocValuesQueryTests.java new file mode 100644 index 0000000000000..57d4d7563e6d3 --- /dev/null +++ b/server/src/test/java/org/opensearch/search/query/Bitmap64DocValuesQueryTests.java @@ -0,0 +1,149 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.LongField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Weight; +import org.apache.lucene.store.Directory; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.roaringbitmap.longlong.Roaring64NavigableMap; + +import static org.opensearch.search.query.Bitmap64IndexQueryTests.getMatchingValues; + +public class Bitmap64DocValuesQueryTests extends OpenSearchTestCase { + private Directory dir; + private IndexWriter w; + private DirectoryReader reader; + private IndexSearcher searcher; + + @Before + public void initSearcher() throws IOException { + dir = newDirectory(); + w = new IndexWriter(dir, newIndexWriterConfig()); + } + + @After + public void closeAllTheReaders() throws IOException { + reader.close(); + w.close(); + dir.close(); + } + + public void testScore() throws IOException { + Document d = new Document(); + d.add(new LongField("product_id", 1L, Field.Store.NO)); + w.addDocument(d); + + d = new Document(); + d.add(new LongField("product_id", 2L, Field.Store.NO)); + w.addDocument(d); + + d = new Document(); + d.add(new LongField("product_id", 3L, Field.Store.NO)); + w.addDocument(d); + + d = new Document(); + d.add(new LongField("product_id", 4L, Field.Store.NO)); + w.addDocument(d); + + w.commit(); + reader = DirectoryReader.open(w); + searcher = newSearcher(reader); + + Roaring64NavigableMap bitmap = new Roaring64NavigableMap(true); + bitmap.addLong(1L); + bitmap.addLong(4L); + Bitmap64DocValuesQuery query = new Bitmap64DocValuesQuery("product_id", bitmap); + + Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1f); + + List actual = getMatchingValues(weight, searcher.getIndexReader()); + List expected = List.of(1L, 4L); + assertEquals(expected, actual); + } + + public void testScoreMultiValues() throws IOException { + Document d = new Document(); + d.add(new LongField("product_id", 1L, Field.Store.NO)); + w.addDocument(d); + + d = new Document(); + d.add(new LongField("product_id", 2L, Field.Store.NO)); + d.add(new LongField("product_id", 3L, Field.Store.NO)); + w.addDocument(d); + + d = new Document(); + d.add(new LongField("product_id", 3L, Field.Store.NO)); + w.addDocument(d); + + d = new Document(); + d.add(new LongField("product_id", 4L, Field.Store.NO)); + w.addDocument(d); + + w.commit(); + reader = DirectoryReader.open(w); + searcher = newSearcher(reader); + + Roaring64NavigableMap bitmap = new Roaring64NavigableMap(true); + bitmap.addLong(3L); + Bitmap64DocValuesQuery query = new Bitmap64DocValuesQuery("product_id", bitmap); + + Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1f); + + Set actual = new HashSet<>(getMatchingValues(weight, searcher.getIndexReader())); + Set expected = Set.of(2L, 3L); + assertEquals(expected, actual); + } + + public void testScoreLargeValues() throws IOException { + long largeVal1 = Integer.MAX_VALUE + 100L; + long largeVal2 = Integer.MAX_VALUE + 200L; + + Document d = new Document(); + d.add(new LongField("product_id", largeVal1, Field.Store.NO)); + w.addDocument(d); + + d = new Document(); + d.add(new LongField("product_id", largeVal2, Field.Store.NO)); + w.addDocument(d); + + d = new Document(); + d.add(new LongField("product_id", 1L, Field.Store.NO)); + w.addDocument(d); + + w.commit(); + reader = DirectoryReader.open(w); + searcher = newSearcher(reader); + + Roaring64NavigableMap bitmap = new Roaring64NavigableMap(true); + bitmap.addLong(largeVal1); + bitmap.addLong(largeVal2); + Bitmap64DocValuesQuery query = new Bitmap64DocValuesQuery("product_id", bitmap); + + Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1f); + + List actual = getMatchingValues(weight, searcher.getIndexReader()); + List expected = List.of(largeVal1, largeVal2); + assertEquals(expected, actual); + } +} diff --git a/server/src/test/java/org/opensearch/search/query/Bitmap64IndexQueryTests.java b/server/src/test/java/org/opensearch/search/query/Bitmap64IndexQueryTests.java new file mode 100644 index 0000000000000..6ad6d9d700846 --- /dev/null +++ b/server/src/test/java/org/opensearch/search/query/Bitmap64IndexQueryTests.java @@ -0,0 +1,370 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.query; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.LongField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.Weight; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.util.TestUtil; +import org.opensearch.common.Randomness; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Random; +import java.util.Set; + +import org.roaringbitmap.longlong.Roaring64NavigableMap; + +public class Bitmap64IndexQueryTests extends OpenSearchTestCase { + private Directory dir; + private IndexWriter w; + private DirectoryReader reader; + private IndexSearcher searcher; + + @Before + public void initSearcher() throws IOException { + dir = newDirectory(); + w = new IndexWriter(dir, newIndexWriterConfig()); + reader = DirectoryReader.open(w); + } + + @After + public void closeAllTheReaders() throws IOException { + reader.close(); + w.close(); + dir.close(); + } + + public void testScore() throws IOException { + Document d = new Document(); + d.add(new LongField("product_id", 1L, Field.Store.NO)); + w.addDocument(d); + + d = new Document(); + d.add(new LongField("product_id", 2L, Field.Store.NO)); + w.addDocument(d); + + d = new Document(); + d.add(new LongField("product_id", 3L, Field.Store.NO)); + w.addDocument(d); + + d = new Document(); + d.add(new LongField("product_id", 4L, Field.Store.NO)); + w.addDocument(d); + + w.commit(); + reader = DirectoryReader.open(w); + searcher = newSearcher(reader); + + Roaring64NavigableMap bitmap = new Roaring64NavigableMap(true); + bitmap.addLong(1L); + bitmap.addLong(4L); + Bitmap64IndexQuery query = new Bitmap64IndexQuery("product_id", bitmap); + + Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1f); + + List actual = getMatchingValues(weight, searcher.getIndexReader()); + List expected = List.of(1L, 4L); + assertEquals(expected, actual); + } + + static List getMatchingValues(Weight weight, IndexReader reader) throws IOException { + List actual = new LinkedList<>(); + for (LeafReaderContext leaf : reader.leaves()) { + SortedNumericDocValues dv = DocValues.getSortedNumeric(leaf.reader(), "product_id"); + Scorer scorer = weight.scorer(leaf); + DocIdSetIterator disi = scorer.iterator(); + int docId; + while ((docId = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + dv.advanceExact(docId); + for (int count = 0; count < dv.docValueCount(); ++count) { + actual.add(dv.nextValue()); + } + } + } + return actual; + } + + public void testScoreMultiValues() throws IOException { + Document d = new Document(); + d.add(new LongField("product_id", 1L, Field.Store.NO)); + w.addDocument(d); + + d = new Document(); + d.add(new LongField("product_id", 2L, Field.Store.NO)); + d.add(new LongField("product_id", 3L, Field.Store.NO)); + w.addDocument(d); + + d = new Document(); + d.add(new LongField("product_id", 3L, Field.Store.NO)); + w.addDocument(d); + + d = new Document(); + d.add(new LongField("product_id", 4L, Field.Store.NO)); + w.addDocument(d); + + w.commit(); + reader = DirectoryReader.open(w); + searcher = newSearcher(reader); + + Roaring64NavigableMap bitmap = new Roaring64NavigableMap(true); + bitmap.addLong(3L); + Bitmap64IndexQuery query = new Bitmap64IndexQuery("product_id", bitmap); + + Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1f); + + Set actual = new HashSet<>(getMatchingValues(weight, searcher.getIndexReader())); + Set expected = Set.of(2L, 3L); + assertEquals(expected, actual); + } + + public void testRandomDocumentsAndQueries() throws IOException { + Random random = Randomness.get(); + int valueRange = 10_000; + + for (int i = 0; i < valueRange + 1; i++) { + Document d = new Document(); + d.add(new LongField("product_id", (long) i, Field.Store.NO)); + w.addDocument(d); + } + + w.commit(); + reader = DirectoryReader.open(w); + searcher = newSearcher(reader); + + Set queryValues = new HashSet<>(); + int numberOfValues = 5; + for (int i = 0; i < numberOfValues; i++) { + long value = random.nextInt(valueRange) + 1L; + queryValues.add(value); + } + Roaring64NavigableMap bitmap = new Roaring64NavigableMap(true); + for (long v : queryValues) { + bitmap.addLong(v); + } + + Bitmap64IndexQuery query = new Bitmap64IndexQuery("product_id", bitmap); + Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1f); + + Set actualSet = new HashSet<>(getMatchingValues(weight, searcher.getIndexReader())); + + List expected = new ArrayList<>(queryValues); + Collections.sort(expected); + List actual = new ArrayList<>(actualSet); + Collections.sort(actual); + assertEquals(expected, actual); + } + + public void testLargeValues() throws IOException { + long largeVal1 = Integer.MAX_VALUE + 100L; + long largeVal2 = Integer.MAX_VALUE + 200L; + long largeVal3 = Integer.MAX_VALUE + 300L; + + Document d = new Document(); + d.add(new LongField("product_id", largeVal1, Field.Store.NO)); + w.addDocument(d); + + d = new Document(); + d.add(new LongField("product_id", largeVal2, Field.Store.NO)); + w.addDocument(d); + + d = new Document(); + d.add(new LongField("product_id", largeVal3, Field.Store.NO)); + w.addDocument(d); + + w.commit(); + reader = DirectoryReader.open(w); + searcher = newSearcher(reader); + + Roaring64NavigableMap bitmap = new Roaring64NavigableMap(true); + bitmap.addLong(largeVal1); + bitmap.addLong(largeVal3); + Bitmap64IndexQuery query = new Bitmap64IndexQuery("product_id", bitmap); + + Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1f); + + List actual = getMatchingValues(weight, searcher.getIndexReader()); + List expected = List.of(largeVal1, largeVal3); + assertEquals(expected, actual); + } + + public void testNegativeLongs() throws IOException { + Document d = new Document(); + d.add(new LongField("product_id", -100L, Field.Store.NO)); + w.addDocument(d); + + d = new Document(); + d.add(new LongField("product_id", 0L, Field.Store.NO)); + w.addDocument(d); + + d = new Document(); + d.add(new LongField("product_id", 100L, Field.Store.NO)); + w.addDocument(d); + + w.commit(); + reader = DirectoryReader.open(w); + searcher = newSearcher(reader); + + Roaring64NavigableMap bitmap = new Roaring64NavigableMap(true); + bitmap.addLong(-100L); + bitmap.addLong(100L); + Bitmap64IndexQuery query = new Bitmap64IndexQuery("product_id", bitmap); + + Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1f); + + List actual = getMatchingValues(weight, searcher.getIndexReader()); + List expected = List.of(-100L, 100L); + assertEquals(expected, actual); + } + + public void testBoundaryValues() throws IOException { + Document d = new Document(); + d.add(new LongField("product_id", Long.MIN_VALUE, Field.Store.NO)); + w.addDocument(d); + + d = new Document(); + d.add(new LongField("product_id", 0L, Field.Store.NO)); + w.addDocument(d); + + d = new Document(); + d.add(new LongField("product_id", Long.MAX_VALUE, Field.Store.NO)); + w.addDocument(d); + + w.commit(); + reader = DirectoryReader.open(w); + searcher = newSearcher(reader); + + Roaring64NavigableMap bitmap = new Roaring64NavigableMap(true); + bitmap.addLong(Long.MIN_VALUE); + bitmap.addLong(Long.MAX_VALUE); + Bitmap64IndexQuery query = new Bitmap64IndexQuery("product_id", bitmap); + + Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1f); + + List actual = getMatchingValues(weight, searcher.getIndexReader()); + List expected = List.of(Long.MIN_VALUE, Long.MAX_VALUE); + assertEquals(expected, actual); + } + + public void testCheckArgsNullBitmap() { + assertThrows(IllegalArgumentException.class, () -> Bitmap64IndexQuery.checkArgs("field", null)); + } + + public void testCheckArgsNullField() { + Roaring64NavigableMap bitmap = new Roaring64NavigableMap(true); + assertThrows(IllegalArgumentException.class, () -> Bitmap64IndexQuery.checkArgs(null, bitmap)); + } + + public void testCheckArgsWithNullBitmap() { + assertThrows(IllegalArgumentException.class, () -> { Bitmap64IndexQuery.checkArgs("product_id", null); }); + } + + public void testCheckArgsWithNullFieldAndBitmap() { + IllegalArgumentException exception = expectThrows( + IllegalArgumentException.class, + () -> { Bitmap64IndexQuery.checkArgs(null, null); } + ); + assertEquals("field must not be null", exception.getMessage()); + } + + public void testCreateWeight() throws IOException { + Document d = new Document(); + d.add(new LongField("product_id", 4L, Field.Store.NO)); + w.addDocument(d); + + w.commit(); + reader = DirectoryReader.open(w); + searcher = newSearcher(reader); + Roaring64NavigableMap bitmap = new Roaring64NavigableMap(true); + bitmap.addLong(1L); + Bitmap64IndexQuery query = new Bitmap64IndexQuery("product_id", bitmap); + Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, 1f); + assertNotNull(weight); + Scorer scorer = weight.scorer(reader.leaves().get(0)); + assertNotNull(scorer); + ScorerSupplier supplier = weight.scorerSupplier(reader.leaves().get(0)); + assertNotNull(supplier); + long cost = supplier.cost(); + assertEquals(20, cost); + } + + public void testRewrite() throws IOException { + Roaring64NavigableMap bitmap = new Roaring64NavigableMap(true); + Bitmap64IndexQuery query = new Bitmap64IndexQuery("product_id", bitmap); + assertEquals(new MatchNoDocsQuery(), query.rewrite(searcher)); + } + + public void testPointVisitor() throws IOException { + w.close(); + w = new IndexWriter(dir, new IndexWriterConfig().setCodec(TestUtil.getDefaultCodec())); + + for (int i = 0; i < 512 + 1; i++) { + Document d = new Document(); + d.add(new LongField("product_id", 1L, Field.Store.NO)); + w.addDocument(d); + } + + for (int i = 0; i < 256 + 1; i++) { + Document d = new Document(); + d.add(new LongField("product_id", 2L, Field.Store.NO)); + w.addDocument(d); + } + + for (int i = 0; i < 256 + 1; i++) { + Document d = new Document(); + d.add(new LongField("product_id", 3L, Field.Store.NO)); + w.addDocument(d); + } + + for (int i = 0; i < 512 + 1; i++) { + Document d = new Document(); + d.add(new LongField("product_id", 4L, Field.Store.NO)); + w.addDocument(d); + } + + w.commit(); + reader = DirectoryReader.open(w); + searcher = newSearcher(reader); + + Roaring64NavigableMap bitmap = new Roaring64NavigableMap(true); + bitmap.addLong(0L); + bitmap.addLong(1L); + bitmap.addLong(2L); + bitmap.addLong(3L); + bitmap.addLong(5L); + Bitmap64IndexQuery query = new Bitmap64IndexQuery("product_id", bitmap); + Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1f); + + Set actual = new HashSet<>(getMatchingValues(weight, searcher.getIndexReader())); + Set expected = Set.of(1L, 2L, 3L); + assertEquals(expected, actual); + } +}