Field collapsing supports search_after

CHANGELOG.md

@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add a dynamic setting to change skip_cache_factor and min_frequency for querycache ([#18351](https://github.com/opensearch-project/OpenSearch/issues/18351))
 - Add overload constructor for Translog to accept Channel Factory as a parameter ([#18918](https://github.com/opensearch-project/OpenSearch/pull/18918))
 - Add subdirectory-aware store module with recovery support ([#19132](https://github.com/opensearch-project/OpenSearch/pull/19132))
+- Field collapsing supports search_after ([#19261](https://github.com/opensearch-project/OpenSearch/pull/19261))
 - Add a dynamic cluster setting to control the enablement of the merged segment warmer ([#18929](https://github.com/opensearch-project/OpenSearch/pull/18929))
 - Publish transport-grpc-spi exposing QueryBuilderProtoConverter and QueryBuilderProtoConverterRegistry ([#18949](https://github.com/opensearch-project/OpenSearch/pull/18949))
 - Support system generated search pipeline. ([#19128](https://github.com/opensearch-project/OpenSearch/pull/19128))

rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml

@@ -245,20 +245,6 @@ setup:
         body:
           collapse: { field: numeric_group }
 
----
-"field collapsing and search_after":
-
-  - do:
-      catch:      /cannot use \`collapse\` in conjunction with \`search_after\`/
-      search:
-        allow_partial_search_results: false
-        rest_total_hits_as_int: true
-        index: test
-        body:
-          collapse: { field: numeric_group }
-          search_after: [6]
-          sort: [{ sort: desc }]
-
 ---
 "field collapsing and rescore":
 
@@ -610,3 +596,334 @@ setup:
   - match: { hits.hits.0._source.marker: 'doc1' }
   - match: { hits.hits.1._id: '2' }
   - match: { hits.hits.1._source.marker: 'doc2' }
+
+
+---
+"field collapsing with search_after - basic functionality":
+  - skip:
+      version: " - 3.2.99"
+      reason: Introduced in 3.3.0
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: numeric_group }
+          sort: [{ numeric_group: asc }]
+          size: 10
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 3 }
+  - match: {hits.hits.0.fields.numeric_group: [1] }
+  - match: {hits.hits.1.fields.numeric_group: [3] }
+  - match: {hits.hits.2.fields.numeric_group: [25] }
+
+  - do:
+      catch: /collapse field and sort field must be the same when use `collapse` in conjunction with `search_after`/
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: numeric_group }
+          sort: [{ sort: desc }]
+          search_after: [10]
+  - do:
+      catch: /collapse field and sort field must be the same when use `collapse` in conjunction with `search_after`/
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: numeric_group }
+          sort: [{ numeric_group: desc }, { sort: asc }]
+          search_after: [25, 10]
+
+  # Test asc, first page
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: numeric_group }
+          sort: [{ numeric_group: asc }]
+          search_after: [1]
+          size: 10
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 2 }
+  - match: {hits.hits.0.fields.numeric_group: [3] }
+  - match: {hits.hits.1.fields.numeric_group: [25] }
+
+  # Test asc, second page
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: numeric_group }
+          sort: [{ numeric_group: asc }]
+          search_after: [3]
+          size: 2
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 1 }
+  - match: {hits.hits.0.fields.numeric_group: [25] }
+
+  # Test asc, no result
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: numeric_group }
+          sort: [{ numeric_group: asc }]
+          search_after: [999]
+          size: 10
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 0 }
+
+  # Test desc, first page
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: numeric_group }
+          sort: [{ numeric_group: desc }]
+          size: 1
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 1 }
+  - match: {hits.hits.0.fields.numeric_group: [25] }
+  - set: { hits.hits.0.sort.0: last_sort_value }
+
+  # Test desc, second page
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: numeric_group }
+          sort: [{ numeric_group: desc }]
+          search_after: [$last_sort_value]
+          size: 1
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 1 }
+  - match: {hits.hits.0.fields.numeric_group: [3] }
+  - set: { hits.hits.0.sort.0: last_sort_value }
+
+  # Test desc, third page
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: numeric_group }
+          sort: [{ numeric_group: desc }]
+          search_after: [$last_sort_value]
+          size: 1
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 1 }
+  - match: {hits.hits.0.fields.numeric_group: [1] }
+  - set: { hits.hits.0.sort.0: last_sort_value }
+
+  # Test desc, no result
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: numeric_group }
+          sort: [{ numeric_group: desc }]
+          search_after: [$last_sort_value]
+          size: 1
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 0 }
+
+  # test on keyword field
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: tag }
+          sort: [{ tag: asc }]
+          size: 1
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 1 }
+  - match: {hits.hits.0.fields.tag: ["A"] }
+
+  # Search after "A"
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: tag }
+          sort: [{ tag: asc }]
+          search_after: ["A"]
+          size: 1
+
+---
+"field collapsing with search_after - concurrent segment search enabled":
+  - skip:
+      version: " - 3.2.99"
+      reason: Introduced in 3.3.0
+  - do:
+      indices.put_settings:
+        index: test
+        body:
+          index.search.concurrent_segment_search.mode: 'all'
+
+  # Test asc, first page
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: numeric_group }
+          sort: [{ numeric_group: asc }]
+          search_after: [1]
+          size: 10
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 2 }
+  - match: {hits.hits.0.fields.numeric_group: [3] }
+  - match: {hits.hits.1.fields.numeric_group: [25] }
+
+  # Test asc, second page
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: numeric_group }
+          sort: [{ numeric_group: asc }]
+          search_after: [3]
+          size: 2
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 1 }
+  - match: {hits.hits.0.fields.numeric_group: [25] }
+
+  # Test asc, no result
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: numeric_group }
+          sort: [{ numeric_group: asc }]
+          search_after: [999]
+          size: 10
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 0 }
+
+  # Test desc, first page
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: numeric_group }
+          sort: [{ numeric_group: desc }]
+          size: 1
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 1 }
+  - match: {hits.hits.0.fields.numeric_group: [25] }
+  - set: { hits.hits.0.sort.0: last_sort_value }
+
+  # Test desc, second page
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: numeric_group }
+          sort: [{ numeric_group: desc }]
+          search_after: [$last_sort_value]
+          size: 1
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 1 }
+  - match: {hits.hits.0.fields.numeric_group: [3] }
+  - set: { hits.hits.0.sort.0: last_sort_value }
+
+  # Test desc, third page
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: numeric_group }
+          sort: [{ numeric_group: desc }]
+          search_after: [$last_sort_value]
+          size: 1
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 1 }
+  - match: {hits.hits.0.fields.numeric_group: [1] }
+  - set: { hits.hits.0.sort.0: last_sort_value }
+
+  # Test desc, no result
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: numeric_group }
+          sort: [{ numeric_group: desc }]
+          search_after: [$last_sort_value]
+          size: 1
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 0 }
+
+  # test on keyword field
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: tag }
+          sort: [{ tag: asc }]
+          size: 1
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 1 }
+  - match: {hits.hits.0.fields.tag: ["A"] }
+
+  # Search after "A"
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: tag }
+          sort: [{ tag: asc }]
+          search_after: ["A"]
+          size: 1
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 1 }
+  - match: {hits.hits.0.fields.tag: ["B"] }
+
+  # Search after "B"
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test
+        body:
+          collapse: { field: tag }
+          sort: [{ tag: asc }]
+          search_after: ["B"]
+          size: 1
+
+  - match: {hits.total: 6 }
+  - length: {hits.hits: 0 }