Fallback to v1 engine for pagination

integ-test/src/test/java/org/opensearch/sql/legacy/CursorIT.java

@@ -123,11 +123,16 @@ public void validNumberOfPages() throws IOException {
     String selectQuery = StringUtils.format("SELECT firstname, state FROM %s", TEST_INDEX_ACCOUNT);
     JSONObject response = new JSONObject(executeFetchQuery(selectQuery, 50, JDBC));
     String cursor = response.getString(CURSOR);
+    verifyIsV1Cursor(cursor);
+
     int pageCount = 1;
 
     while (!cursor.isEmpty()) { //this condition also checks that there is no cursor on last page
       response = executeCursorQuery(cursor);
       cursor = response.optString(CURSOR);
+      if (!cursor.isEmpty()) {
+        verifyIsV1Cursor(cursor);
+      }
       pageCount++;
     }
 
@@ -136,12 +141,16 @@ public void validNumberOfPages() throws IOException {
     // using random value here, with fetch size of 28 we should get 36 pages (ceil of 1000/28)
     response = new JSONObject(executeFetchQuery(selectQuery, 28, JDBC));
     cursor = response.getString(CURSOR);
+    verifyIsV1Cursor(cursor);
     System.out.println(response);
     pageCount = 1;
 
     while (!cursor.isEmpty()) {
       response = executeCursorQuery(cursor);
       cursor = response.optString(CURSOR);
+      if (!cursor.isEmpty()) {
+        verifyIsV1Cursor(cursor);
+      }
       pageCount++;
     }
     assertThat(pageCount, equalTo(36));
@@ -223,6 +232,7 @@ public void testCursorWithPreparedStatement() throws IOException {
         "}", TestsConstants.TEST_INDEX_ACCOUNT));
 
     assertTrue(response.has(CURSOR));
+    verifyIsV1Cursor(response.getString(CURSOR));
   }
 
   @Test
@@ -244,11 +254,13 @@ public void testRegressionOnDateFormatChange() throws IOException {
         StringUtils.format("SELECT login_time FROM %s LIMIT 500", TEST_INDEX_DATE_TIME);
     JSONObject response = new JSONObject(executeFetchQuery(selectQuery, 1, JDBC));
     String cursor = response.getString(CURSOR);
+    verifyIsV1Cursor(cursor);
     actualDateList.add(response.getJSONArray(DATAROWS).getJSONArray(0).getString(0));
 
     while (!cursor.isEmpty()) {
       response = executeCursorQuery(cursor);
       cursor = response.optString(CURSOR);
+      verifyIsV1Cursor(cursor);
       actualDateList.add(response.getJSONArray(DATAROWS).getJSONArray(0).getString(0));
     }
 
@@ -274,7 +286,6 @@ public void defaultBehaviorWhenCursorSettingIsDisabled() throws IOException {
     query = StringUtils.format("SELECT firstname, email, state FROM %s", TEST_INDEX_ACCOUNT);
     response = new JSONObject(executeFetchQuery(query, 100, JDBC));
     assertTrue(response.has(CURSOR));
-
     wipeAllClusterSettings();
   }
 
@@ -305,12 +316,14 @@ public void testDefaultFetchSizeFromClusterSettings() throws IOException {
     JSONObject response = new JSONObject(executeFetchLessQuery(query, JDBC));
     JSONArray datawRows = response.optJSONArray(DATAROWS);
     assertThat(datawRows.length(), equalTo(1000));
+    verifyIsV1Cursor(response.getString(CURSOR));
 
     updateClusterSettings(new ClusterSetting(TRANSIENT, "opensearch.sql.cursor.fetch_size", "786"));
     response = new JSONObject(executeFetchLessQuery(query, JDBC));
     datawRows = response.optJSONArray(DATAROWS);
     assertThat(datawRows.length(), equalTo(786));
     assertTrue(response.has(CURSOR));
+    verifyIsV1Cursor(response.getString(CURSOR));
 
     wipeAllClusterSettings();
   }
@@ -323,11 +336,12 @@ public void testCursorCloseAPI() throws IOException {
         "SELECT firstname, state FROM %s WHERE balance > 100 and age < 40", TEST_INDEX_ACCOUNT);
     JSONObject result = new JSONObject(executeFetchQuery(selectQuery, 50, JDBC));
     String cursor = result.getString(CURSOR);
-
+    verifyIsV1Cursor(cursor);
     // Retrieving next 10 pages out of remaining 19 pages
     for (int i = 0; i < 10; i++) {
       result = executeCursorQuery(cursor);
       cursor = result.optString(CURSOR);
+      verifyIsV1Cursor(cursor);
     }
     //Closing the cursor
     JSONObject closeResp = executeCursorCloseQuery(cursor);
@@ -386,12 +400,14 @@ public void respectLimitPassedInSelectClause() throws IOException {
         StringUtils.format("SELECT age, balance FROM %s LIMIT %s", TEST_INDEX_ACCOUNT, limit);
     JSONObject response = new JSONObject(executeFetchQuery(selectQuery, 50, JDBC));
     String cursor = response.getString(CURSOR);
+    verifyIsV1Cursor(cursor);
     int actualDataRowCount = response.getJSONArray(DATAROWS).length();
     int pageCount = 1;
 
     while (!cursor.isEmpty()) {
       response = executeCursorQuery(cursor);
       cursor = response.optString(CURSOR);
+      verifyIsV1Cursor(cursor);
       actualDataRowCount += response.getJSONArray(DATAROWS).length();
       pageCount++;
     }
@@ -432,10 +448,12 @@ public void verifyWithAndWithoutPaginationResponse(String sqlQuery, String curso
     response.optJSONArray(DATAROWS).forEach(dataRows::put);
 
     String cursor = response.getString(CURSOR);
+    verifyIsV1Cursor(cursor);
     while (!cursor.isEmpty()) {
       response = executeCursorQuery(cursor);
       response.optJSONArray(DATAROWS).forEach(dataRows::put);
       cursor = response.optString(CURSOR);
+      verifyIsV1Cursor(cursor);
     }
 
     verifySchema(withoutCursorResponse.optJSONArray(SCHEMA),
@@ -465,6 +483,13 @@ public String executeFetchAsStringQuery(String query, String fetchSize, String r
     return responseString;
   }
 
+  private void verifyIsV1Cursor(String cursor) {
+    if (cursor.isEmpty()) {
+      return;
+    }
+    assertTrue("The cursor '" + cursor + "' is not from v1 engine.", cursor.startsWith("d:"));
+  }
+
   private String makeRequest(String query, String fetch_size) {
     return String.format("{" +
         "  \"fetch_size\": \"%s\"," +

integ-test/src/test/java/org/opensearch/sql/sql/PaginationFallbackIT.java

@@ -0,0 +1,135 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.sql;
+
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX;
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ONLINE;
+import static org.opensearch.sql.util.TestUtils.verifyIsV1Cursor;
+import static org.opensearch.sql.util.TestUtils.verifyIsV2Cursor;
+
+import java.io.IOException;
+import org.json.JSONObject;
+import org.junit.Test;
+import org.opensearch.sql.legacy.SQLIntegTestCase;
+import org.opensearch.sql.util.TestUtils;
+
+public class PaginationFallbackIT extends SQLIntegTestCase {
+  @Override
+  public void init() throws IOException {
+    loadIndex(Index.PHRASE);
+    loadIndex(Index.ONLINE);
+  }
+
+  @Test
+  public void testWhereClause() throws IOException {
+    var response = executeQueryTemplate("SELECT * FROM %s WHERE 1 = 1", TEST_INDEX_ONLINE);
+    verifyIsV1Cursor(response);
+  }
+
+  @Test
+  public void testSelectAll() throws IOException {
+    var response = executeQueryTemplate("SELECT * FROM %s", TEST_INDEX_ONLINE);
+    verifyIsV2Cursor(response);
+  }
+
+  @Test
+  public void testSelectWithOpenSearchFuncInFilter() throws IOException {
+    var response = executeQueryTemplate(
+        "SELECT * FROM %s WHERE `11` = match_phrase('96')", TEST_INDEX_ONLINE);
+    verifyIsV1Cursor(response);
+  }
+
+  @Test
+  public void testSelectWithHighlight() throws IOException {
+    var response = executeQueryTemplate(
+        "SELECT highlight(`11`) FROM %s WHERE match_query(`11`, '96')", TEST_INDEX_ONLINE);
+    // As of 2023-03-08, WHERE clause sends the query to legacy engine and legacy engine
+    // does not support highlight as an expression.
+    assertTrue(response.has("error"));
+  }
+
+  @Test
+  public void testSelectWithFullTextSearch() throws IOException {
+    var response = executeQueryTemplate(
+        "SELECT * FROM %s WHERE match_phrase(`11`, '96')", TEST_INDEX_ONLINE);
+    verifyIsV1Cursor(response);
+  }
+
+  @Test
+  public void testSelectFromIndexWildcard() throws IOException {
+    var response = executeQueryTemplate("SELECT * FROM %s*", TEST_INDEX);
+    verifyIsV2Cursor(response);
+  }
+
+  @Test
+  public void testSelectFromDataSource() throws IOException {
+    var response = executeQueryTemplate("SELECT * FROM @opensearch.%s",
+        TEST_INDEX_ONLINE);
+    verifyIsV2Cursor(response);
+  }
+
+  @Test
+  public void testSelectColumnReference() throws IOException {
+    var response = executeQueryTemplate("SELECT `107` from %s", TEST_INDEX_ONLINE);
+    verifyIsV1Cursor(response);
+  }
+
+  @Test
+  public void testSubquery() throws IOException {
+    var response = executeQueryTemplate("SELECT `107` from (SELECT * FROM %s)",
+        TEST_INDEX_ONLINE);
+    verifyIsV1Cursor(response);
+  }
+
+  @Test
+  public void testSelectExpression() throws IOException {
+    var response = executeQueryTemplate("SELECT 1 + 1 - `107` from %s",
+        TEST_INDEX_ONLINE);
+    verifyIsV1Cursor(response);
+  }
+
+  @Test
+  public void testGroupBy() throws IOException {
+    // GROUP BY is not paged by either engine.
+     var response = executeQueryTemplate("SELECT * FROM %s GROUP BY `107`",
+      TEST_INDEX_ONLINE);
+    TestUtils.verifyNoCursor(response);
+  }
+
+  @Test
+  public void testGroupByHaving() throws IOException {
+    // GROUP BY is not paged by either engine.
+    var response = executeQueryTemplate("SELECT * FROM %s GROUP BY `107` HAVING `107` > 400",
+        TEST_INDEX_ONLINE);
+    TestUtils.verifyNoCursor(response);
+  }
+
+  @Test
+  public void testLimit() throws IOException {
+    var response = executeQueryTemplate("SELECT * FROM %s LIMIT 8", TEST_INDEX_ONLINE);
+    verifyIsV1Cursor(response);
+  }
+
+  @Test
+  public void testLimitOffset() throws IOException {
+    var response = executeQueryTemplate("SELECT * FROM %s LIMIT 8 OFFSET 4",
+        TEST_INDEX_ONLINE);
+    verifyIsV1Cursor(response);
+  }
+
+  @Test
+  public void testOrderBy() throws IOException {
+    var response = executeQueryTemplate("SELECT * FROM %s ORDER By `107`",
+        TEST_INDEX_ONLINE);
+    verifyIsV1Cursor(response);
+  }
+
+  private JSONObject executeQueryTemplate(String queryTemplate, String index) throws IOException {
+    var query = String.format(queryTemplate, index);
+    return new JSONObject(executeFetchQuery(query, 4, "jdbc"));
+  }
+
+}

integ-test/src/test/java/org/opensearch/sql/sql/PaginationIT.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.sql;
+
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS;
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ONLINE;
+
+import java.io.IOException;
+import org.json.JSONObject;
+import org.junit.Test;
+import org.opensearch.sql.legacy.SQLIntegTestCase;
+import org.opensearch.sql.util.TestUtils;
+
+public class PaginationIT extends SQLIntegTestCase {
+  @Override
+  public void init() throws IOException {
+    loadIndex(Index.CALCS);
+    loadIndex(Index.ONLINE);
+  }
+
+  @Test
+  public void testSmallDataSet() throws IOException {
+    var query = "SELECT * from " + TEST_INDEX_CALCS;
+    var response = new JSONObject(executeFetchQuery(query, 4, "jdbc"));
+    assertTrue(response.has("cursor"));
+    assertEquals(4, response.getInt("size"));
+    TestUtils.verifyIsV2Cursor(response);
+  }
+
+  @Test
+  public void testLargeDataSetV1() throws IOException {
+    var v1query = "SELECT * from " + TEST_INDEX_ONLINE + " WHERE 1 = 1";
+    var v1response = new JSONObject(executeFetchQuery(v1query, 4, "jdbc"));
+    assertEquals(4, v1response.getInt("size"));
+    TestUtils.verifyIsV1Cursor(v1response);
+  }
+
+  @Test
+  public void testLargeDataSetV2() throws IOException {
+    var query = "SELECT * from " + TEST_INDEX_ONLINE;
+    var response = new JSONObject(executeFetchQuery(query, 4, "jdbc"));
+    assertEquals(4, response.getInt("size"));
+    TestUtils.verifyIsV2Cursor(response);
+  }
+}

integ-test/src/test/java/org/opensearch/sql/util/TestUtils.java

@@ -7,6 +7,7 @@
 package org.opensearch.sql.util;
 
 import static com.google.common.base.Strings.isNullOrEmpty;
+import static org.junit.Assert.assertTrue;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -839,4 +840,26 @@ public static List<List<String>> getPermutations(final List<String> items) {
 
     return result;
   }
+
+  public static void verifyIsV1Cursor(JSONObject response) {
+    verifyCursor(response, List.of("d:", "j:", "a:"), "v1");
+  }
+
+
+  public static void verifyIsV2Cursor(JSONObject response) {
+    verifyCursor(response, List.of("n:"), "v2");
+  }
+
+  private static void verifyCursor(JSONObject response, List<String> validCursorPrefix, String engineName) {
+      assertTrue("'cursor' property does not exist", response.has("cursor"));
+
+      var cursor = response.getString("cursor");
+      assertTrue("'cursor' property is empty", !cursor.isEmpty());
+      assertTrue("The cursor '" + cursor + "' is not from " + engineName + " engine.",
+          validCursorPrefix.stream().anyMatch(cursor::startsWith));
+    }
+
+  public static void verifyNoCursor(JSONObject response) {
+    assertTrue(!response.has("cursor"));
+  }
 }