Add bitmap64 query support

CHANGELOG.md

@@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased 3.x]
 ### Added
+- Add bitmap64 query support ([#20606](https://github.com/opensearch-project/OpenSearch/pull/20606))
 - Add ProfilingWrapper interface for plugin access to delegates in profiling decorators ([#20607](https://github.com/opensearch-project/OpenSearch/pull/20607))
 - Support expected cluster name with validation in CCS Sniff mode ([#20532](https://github.com/opensearch-project/OpenSearch/pull/20532))
 - Choose the best performing node when writing with append-only index ([#20065](https://github.com/opensearch-project/OpenSearch/pull/20065))

rest-api-spec/src/main/resources/rest-api-spec/test/search/381_bitmap_filtering_long.yml

@@ -0,0 +1,99 @@
+---
+setup:
+  - skip:
+      version: " - 3.5.99"
+      reason: Bitmap filtering for long fields is available in 3.6 and later.
+
+  - do:
+      indices.create:
+        index: employees
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            properties:
+              employee_id:
+                type: long
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - { "index": { "_index": "employees", "_id": "1" } }
+          - { "name": "Alice Smith", "employee_id": 1000000000001 }
+          - { "index": { "_index": "employees", "_id": "2" } }
+          - { "name": "Bob Johnson", "employee_id": 2000000000002 }
+          - { "index": { "_index": "employees", "_id": "3" } }
+          - { "name": "Charlie Brown", "employee_id": 3000000000003 }
+
+  - do:
+      indices.create:
+        index: departments
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            properties:
+              members:
+                type: binary
+                store: true
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - { "index": { "_index": "departments", "_id": "201" } }
+          - { "members": "AgAAAAAAAADoAAAAOjAAAAEAAACl1AAAEAAAAAEQ0QEAADowAAABAAAASqkAABAAAAACIA==" }
+          - { "index": { "_index": "departments", "_id": "202" } }
+          - { "members": "AQAAAAAAAADoAAAAOjAAAAEAAACl1AAAEAAAAAEQ" }
+
+  - do:
+      cluster.health:
+        wait_for_status: green
+
+---
+"Terms lookup on a binary field with bitmap (long)":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: employees
+        body: {
+          "query": {
+            "terms": {
+              "employee_id": {
+                "index": "departments",
+                "id": "201",
+                "path": "members",
+                "store": true
+              },
+              "value_type": "bitmap"
+            }
+          }
+        }
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._source.name: Alice Smith }
+  - match: { hits.hits.0._source.employee_id: 1000000000001 }
+  - match: { hits.hits.1._source.name: Bob Johnson }
+  - match: { hits.hits.1._source.employee_id: 2000000000002 }
+
+---
+"Terms query accepting bitmap as value (long)":
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: employees
+        body: {
+          "query": {
+            "terms": {
+              "employee_id": ["AgAAAAAAAADoAAAAOjAAAAEAAACl1AAAEAAAAAEQ0QEAADowAAABAAAASqkAABAAAAACIA=="],
+              "value_type": "bitmap"
+            }
+          }
+        }
+  - match: { hits.total: 2 }
+  - match: { hits.hits.0._source.name: Alice Smith }
+  - match: { hits.hits.0._source.employee_id: 1000000000001 }
+  - match: { hits.hits.1._source.name: Bob Johnson }
+  - match: { hits.hits.1._source.employee_id: 2000000000002 }

server/src/internalClusterTest/java/org/opensearch/search/query/SearchQueryIT.java

@@ -84,6 +84,8 @@
 import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase;
 import org.opensearch.test.junit.annotations.TestIssueLogging;
 
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
 import java.io.IOException;
 import java.io.Reader;
 import java.nio.ByteBuffer;
@@ -102,6 +104,7 @@
 import java.util.regex.Pattern;
 
 import org.roaringbitmap.RoaringBitmap;
+import org.roaringbitmap.longlong.Roaring64NavigableMap;
 
 import static java.util.Collections.singletonMap;
 import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
@@ -1197,6 +1200,88 @@ public void testTermsQueryWithBitmapDocValuesQuery() throws Exception {
         assertSearchHits(searchResponse, "1", "3", "4");
     }
 
+    public void testTermsQueryWithBitmap64DocValuesQuery() throws Exception {
+        assertAcked(
+            prepareCreate("employees").setMapping(
+                jsonBuilder().startObject()
+                    .startObject("properties")
+                    .startObject("employee_id")
+                    .field("type", "long")
+                    .field("index", false)
+                    .endObject()
+                    .endObject()
+                    .endObject()
+            )
+        );
+        indexRandom(
+            true,
+            client().prepareIndex("employees").setId("1").setSource("employee_id", 1000000000001L),
+            client().prepareIndex("employees").setId("2").setSource("employee_id", 2000000000002L),
+            client().prepareIndex("employees").setId("3").setSource("employee_id", new long[] { 1000000000001L, 3000000000003L }),
+            client().prepareIndex("employees").setId("4").setSource("employee_id", 4000000000004L)
+        );
+        refresh();
+
+        Roaring64NavigableMap bitmap = new Roaring64NavigableMap();
+        bitmap.addLong(1000000000001L);
+        bitmap.addLong(4000000000004L);
+
+        ByteArrayOutputStream bos = new ByteArrayOutputStream();
+        DataOutputStream dos = new DataOutputStream(bos);
+        bitmap.serializePortable(dos);
+        dos.close();
+
+        BytesArray bitmapBytes = new BytesArray(bos.toByteArray());
+
+        // directly building the terms query builder, so pass in the bitmap value as BytesArray
+        SearchResponse searchResponse = client().prepareSearch("employees")
+            .setQuery(constantScoreQuery(termsQuery("employee_id", bitmapBytes).valueType(TermsQueryBuilder.ValueType.BITMAP)))
+            .get();
+        assertHitCount(searchResponse, 3L);
+        assertSearchHits(searchResponse, "1", "3", "4");
+    }
+
+    public void testTermsQueryWithBitmap64IndexAndDocValues() throws Exception {
+        assertAcked(
+            prepareCreate("employees2").setMapping(
+                jsonBuilder().startObject()
+                    .startObject("properties")
+                    .startObject("employee_id")
+                    .field("type", "long")
+                    // Both index and doc values enabled (default)
+                    .endObject()
+                    .endObject()
+                    .endObject()
+            )
+        );
+
+        indexRandom(
+            true,
+            client().prepareIndex("employees2").setId("1").setSource("employee_id", 1000000000001L),
+            client().prepareIndex("employees2").setId("2").setSource("employee_id", 2000000000002L),
+            client().prepareIndex("employees2").setId("3").setSource("employee_id", 3000000000003L)
+        );
+        refresh();
+
+        Roaring64NavigableMap bitmap = new Roaring64NavigableMap();
+        bitmap.addLong(1000000000001L);
+        bitmap.addLong(3000000000003L);
+
+        ByteArrayOutputStream bos = new ByteArrayOutputStream();
+        try (DataOutputStream dos = new DataOutputStream(bos)) {
+            bitmap.serializePortable(dos);
+        }
+
+        BytesArray bitmapBytes = new BytesArray(bos.toByteArray());
+
+        SearchResponse searchResponse = client().prepareSearch("employees2")
+            .setQuery(constantScoreQuery(termsQuery("employee_id", bitmapBytes).valueType(TermsQueryBuilder.ValueType.BITMAP)))
+            .get();
+
+        assertHitCount(searchResponse, 2L);
+        assertSearchHits(searchResponse, "1", "3");
+    }
+
     public void testTermsLookupFilter() throws Exception {
         assertAcked(prepareCreate("lookup").setMapping("terms", "type=text", "other", "type=text"));
         indexRandomForConcurrentSearch("lookup");

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

@@ -74,9 +74,13 @@
 import org.opensearch.search.approximate.ApproximatePointRangeQuery;
 import org.opensearch.search.approximate.ApproximateScoreQuery;
 import org.opensearch.search.lookup.SearchLookup;
+import org.opensearch.search.query.Bitmap64DocValuesQuery;
+import org.opensearch.search.query.Bitmap64IndexQuery;
 import org.opensearch.search.query.BitmapDocValuesQuery;
 import org.opensearch.search.query.BitmapIndexQuery;
 
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
 import java.io.IOException;
 import java.math.BigInteger;
 import java.nio.ByteBuffer;
@@ -93,6 +97,7 @@
 import java.util.function.Supplier;
 
 import org.roaringbitmap.RoaringBitmap;
+import org.roaringbitmap.longlong.Roaring64NavigableMap;
 
 /**
  * A {@link FieldMapper} for numeric types: byte, short, int, long, float, double and unsigned long.
@@ -1368,6 +1373,29 @@ public Query termsQuery(String field, List<Object> values, boolean hasDocValues,
                 return LongPoint.newSetQuery(field, v);
             }
 
+            @Override
+            public Query bitmapQuery(String field, BytesArray bitmapArray, boolean isSearchable, boolean hasDocValues) {
+                // Extract bytes safely
+                BytesRef ref = bitmapArray.toBytesRef();
+                byte[] bytes = Arrays.copyOfRange(ref.bytes, ref.offset, ref.offset + ref.length);
+
+                Roaring64NavigableMap bitmap64 = new Roaring64NavigableMap();
+                try {
+                    bitmap64.deserializePortable(new DataInputStream(new ByteArrayInputStream(bytes)));
+                } catch (IOException e) {
+                    throw new IllegalArgumentException("Failed to deserialize the 64-bit bitmap.", e);
+                }
+
+                // Note: bitmap64 instance is safely shared between queries as both perform read-only operations
+                if (isSearchable && hasDocValues) {
+                    return new IndexOrDocValuesQuery(new Bitmap64IndexQuery(field, bitmap64), new Bitmap64DocValuesQuery(field, bitmap64));
+                }
+                if (isSearchable) {
+                    return new Bitmap64IndexQuery(field, bitmap64);
+                }
+                return new Bitmap64DocValuesQuery(field, bitmap64);
+            }
+
             @Override
             public Query rangeQuery(
                 String field,