Support for pagination in v2 engine of SELECT * FROM <table> queries (opensearch-project#1666)

v2 SQL engine can now paginate simple queries. Pagination is initiated by setting fetch_size property in the request JSON.

Pagination is implemented using the OpenSearch Scroll API. Please see pagination-v2.md for implementation details.
---------

Signed-off-by: MaxKsyunz <[email protected]>
Signed-off-by: Yury-Fridlyand <[email protected]>
Signed-off-by: Max Ksyunz <[email protected]>
Co-authored-by: Yury-Fridlyand <[email protected]>
Co-authored-by: GabeFernandez310 <[email protected]>
Co-authored-by: Andrew Carbonetto <[email protected]>
Signed-off-by: Mitchell Gale <[email protected]>

Author: 4 people

opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java

@@ -14,7 +14,6 @@
 import static org.opensearch.search.sort.SortOrder.ASC;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
@@ -281,6 +280,10 @@ private List<NestedQueryBuilder> extractNestedQueries(QueryBuilder query) {
     return result;
   }
 
+  public int getMaxResponseSize() {
+    return pageSize == null ? requestedTotalSize : pageSize;
+  }
+
   /**
    * Initialize bool query for push down.
    */

opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/FilterQueryBuilder.java

@@ -14,18 +14,15 @@
 import java.util.Map;
 import java.util.function.BiFunction;
 import lombok.RequiredArgsConstructor;
-import org.apache.lucene.search.join.ScoreMode;
 import org.opensearch.index.query.BoolQueryBuilder;
 import org.opensearch.index.query.QueryBuilder;
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.index.query.ScriptQueryBuilder;
 import org.opensearch.script.Script;
-import org.opensearch.sql.ast.expression.Function;
 import org.opensearch.sql.common.antlr.SyntaxCheckException;
 import org.opensearch.sql.expression.Expression;
 import org.opensearch.sql.expression.ExpressionNodeVisitor;
 import org.opensearch.sql.expression.FunctionExpression;
-import org.opensearch.sql.expression.ReferenceExpression;
 import org.opensearch.sql.expression.function.BuiltinFunctionName;
 import org.opensearch.sql.expression.function.FunctionName;
 import org.opensearch.sql.opensearch.storage.script.filter.lucene.LikeQuery;

opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java

@@ -384,22 +384,20 @@ void testPushDownNestedWithNestedFilter() {
         .innerHit(new InnerHitBuilder().setFetchSourceContext(
             new FetchSourceContext(true, new String[]{"message.info"}, null)));
 
-    assertEquals(
-        new SearchSourceBuilder()
-            .query(
-                QueryBuilders.boolQuery().filter(
-                    QueryBuilders.boolQuery()
-                        .must(filterQuery)
-                )
-            )
-            .from(DEFAULT_OFFSET)
-            .size(DEFAULT_LIMIT)
-            .timeout(DEFAULT_QUERY_TIMEOUT),
-        requestBuilder.getSourceBuilder());
+    assertSearchSourceBuilder(new SearchSourceBuilder()
+        .query(
+          QueryBuilders.boolQuery().filter(
+            QueryBuilders.boolQuery()
+              .must(filterQuery)
+        )
+      )
+        .from(DEFAULT_OFFSET)
+        .size(DEFAULT_LIMIT)
+        .timeout(DEFAULT_QUERY_TIMEOUT), requestBuilder);
   }
 
   @Test
-  void testPushTypeMapping() {
+  void test_push_type_mapping() {
     Map<String, OpenSearchDataType> typeMapping = Map.of("intA", OpenSearchDataType.of(INTEGER));
     requestBuilder.pushTypeMapping(typeMapping);