Version bump to OpenSearch 3.7 with async QueryPlanExecutor

async-query/src/test/java/org/opensearch/sql/spark/transport/format/CreateAsyncQueryRequestConverterTest.java

@@ -54,7 +54,8 @@ public void fromXContentWithDuplicateFields() throws IOException {
     Assertions.assertTrue(
         illegalArgumentException
             .getMessage()
-            .contains("Error while parsing the request body: Duplicate field 'datasource'"));
+            .contains(
+                "Error while parsing the request body: Duplicate Object property \"datasource\""));
   }
 
   @Test

build.gradle

@@ -6,7 +6,7 @@
 
 buildscript {
     ext {
-        opensearch_version = System.getProperty("opensearch.version", "3.6.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "3.7.0-SNAPSHOT")
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         version_tokens = opensearch_version.tokenize('-')
@@ -127,6 +127,11 @@ allprojects {
         version += "-SNAPSHOT"
     }
 
+    // Path to the analytics-engine plugin ZIP. Override with
+    // `-PanalyticsEngineZip=/path/to/zip` if needed.
+    ext.analyticsEngineZip = project.findProperty('analyticsEngineZip') ?:
+            "${rootDir}/libs/analytics-engine-3.7.0-SNAPSHOT.zip"
+
     plugins.withId('java') {
         java {
             sourceCompatibility = JavaVersion.VERSION_21

core/build.gradle

@@ -33,6 +33,7 @@ plugins {
 }
 
 repositories {
+    mavenLocal()
     mavenCentral()
 }
 
@@ -63,8 +64,12 @@ dependencies {
     }
     api 'org.apache.calcite:calcite-linq4j:1.41.0'
     api project(':common')
-    compileOnly files("${rootDir}/libs/analytics-framework-3.6.0-SNAPSHOT.jar")
-    testImplementation files("${rootDir}/libs/analytics-framework-3.6.0-SNAPSHOT.jar")
+    compileOnly files("${rootDir}/libs/analytics-framework-3.7.0-SNAPSHOT.jar")
+    // Needed because the analytics-framework's QueryPlanExecutor signature uses
+    // org.opensearch.core.action.ActionListener; AnalyticsExecutionEngine references that type.
+    compileOnly group: 'org.opensearch', name: 'opensearch-core', version: "${opensearch_version}"
+    testImplementation files("${rootDir}/libs/analytics-framework-3.7.0-SNAPSHOT.jar")
+    testImplementation group: 'org.opensearch', name: 'opensearch-core', version: "${opensearch_version}"
     implementation "com.github.seancfoley:ipaddress:5.4.2"
     implementation "com.jayway.jsonpath:json-path:2.9.0"
 

core/src/main/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngine.java

@@ -14,6 +14,7 @@
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeField;
 import org.opensearch.analytics.exec.QueryPlanExecutor;
+import org.opensearch.core.action.ActionListener;
 import org.opensearch.sql.ast.statement.ExplainMode;
 import org.opensearch.sql.calcite.CalcitePlanContext;
 import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory;
@@ -70,25 +71,35 @@ public void explain(PhysicalPlan plan, ResponseListener<ExplainResponse> listene
   @Override
   public void execute(
       RelNode plan, CalcitePlanContext context, ResponseListener<QueryResponse> listener) {
-    try {
-      // Record EXECUTE metric before calling listener, because the listener's onResponse
-      // triggers SimpleJsonResponseFormatter which calls QueryProfiling.finish() to snapshot
-      // all metrics. The metric must be written before that snapshot.
-      ProfileMetric execMetric = QueryProfiling.current().getOrCreateMetric(MetricName.EXECUTE);
-      long execStart = System.nanoTime();
-
-      Iterable<Object[]> rows = planExecutor.execute(plan, null);
-
-      List<RelDataTypeField> fields = plan.getRowType().getFieldList();
-      List<ExprValue> results = convertRows(rows, fields);
-      Schema schema = buildSchema(fields);
-
-      execMetric.set(System.nanoTime() - execStart);
-
-      listener.onResponse(new QueryResponse(schema, results, Cursor.None));
-    } catch (Exception e) {
-      listener.onFailure(e);
-    }
+    // QueryPlanExecutor became asynchronous in analytics-framework 3.7 — execution is dispatched
+    // to a worker pool and results arrive on the listener. Record the execute metric in the
+    // listener callback, before delegating to the user-supplied listener, so the metric snapshot
+    // taken by SimpleJsonResponseFormatter sees the correct value.
+    ProfileMetric execMetric = QueryProfiling.current().getOrCreateMetric(MetricName.EXECUTE);
+    long execStart = System.nanoTime();
+
+    planExecutor.execute(
+        plan,
+        null,
+        new ActionListener<>() {
+          @Override
+          public void onResponse(Iterable<Object[]> rows) {
+            try {
+              List<RelDataTypeField> fields = plan.getRowType().getFieldList();
+              List<ExprValue> results = convertRows(rows, fields);
+              Schema schema = buildSchema(fields);
+              execMetric.set(System.nanoTime() - execStart);
+              listener.onResponse(new QueryResponse(schema, results, Cursor.None));
+            } catch (Exception e) {
+              listener.onFailure(e);
+            }
+          }
+
+          @Override
+          public void onFailure(Exception e) {
+            listener.onFailure(e);
+          }
+        });
   }
 
   @Override

core/src/test/java/org/opensearch/sql/executor/analytics/AnalyticsExecutionEngineTest.java

@@ -8,6 +8,9 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -25,6 +28,7 @@
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.opensearch.analytics.exec.QueryPlanExecutor;
+import org.opensearch.core.action.ActionListener;
 import org.opensearch.sql.calcite.CalcitePlanContext;
 import org.opensearch.sql.calcite.SysLimit;
 import org.opensearch.sql.common.response.ResponseListener;
@@ -57,11 +61,34 @@ private static void setSysLimit(CalcitePlanContext context, SysLimit sysLimit) t
     field.set(context, sysLimit);
   }
 
+  /** QueryPlanExecutor became async in analytics-framework 3.7 — stub the listener callback. */
+  @SuppressWarnings("unchecked")
+  private void stubExecutorWith(RelNode relNode, Iterable<Object[]> rows) {
+    doAnswer(
+            inv -> {
+              ((ActionListener<Iterable<Object[]>>) inv.getArgument(2)).onResponse(rows);
+              return null;
+            })
+        .when(mockExecutor)
+        .execute(eq(relNode), any(), any(ActionListener.class));
+  }
+
+  @SuppressWarnings("unchecked")
+  private void stubExecutorWithError(RelNode relNode, Exception error) {
+    doAnswer(
+            inv -> {
+              ((ActionListener<Iterable<Object[]>>) inv.getArgument(2)).onFailure(error);
+              return null;
+            })
+        .when(mockExecutor)
+        .execute(eq(relNode), any(), any(ActionListener.class));
+  }
+
   @Test
   void executeRelNode_basicTypesAndRows() {
     RelNode relNode = mockRelNode("name", SqlTypeName.VARCHAR, "age", SqlTypeName.INTEGER);
     Iterable<Object[]> rows = Arrays.asList(new Object[] {"Alice", 30}, new Object[] {"Bob", 25});
-    when(mockExecutor.execute(relNode, null)).thenReturn(rows);
+    stubExecutorWith(relNode, rows);
 
     QueryResponse response = executeAndCapture(relNode);
     String dump = dumpResponse(response);
@@ -101,7 +128,7 @@ void executeRelNode_numericTypes() {
             "d", SqlTypeName.DOUBLE);
     Iterable<Object[]> rows =
         Collections.singletonList(new Object[] {(byte) 1, (short) 2, 3, 4L, 5.0f, 6.0});
-    when(mockExecutor.execute(relNode, null)).thenReturn(rows);
+    stubExecutorWith(relNode, rows);
 
     QueryResponse response = executeAndCapture(relNode);
     String dump = dumpResponse(response);
@@ -138,7 +165,7 @@ void executeRelNode_temporalTypes() {
     RelNode relNode =
         mockRelNode("dt", SqlTypeName.DATE, "tm", SqlTypeName.TIME, "ts", SqlTypeName.TIMESTAMP);
     Iterable<Object[]> emptyRows = Collections.emptyList();
-    when(mockExecutor.execute(relNode, null)).thenReturn(emptyRows);
+    stubExecutorWith(relNode, emptyRows);
 
     QueryResponse response = executeAndCapture(relNode);
     String dump = dumpResponse(response);
@@ -157,7 +184,7 @@ void executeRelNode_temporalTypes() {
   void executeRelNode_emptyResults() {
     RelNode relNode = mockRelNode("name", SqlTypeName.VARCHAR);
     Iterable<Object[]> emptyRows = Collections.emptyList();
-    when(mockExecutor.execute(relNode, null)).thenReturn(emptyRows);
+    stubExecutorWith(relNode, emptyRows);
 
     QueryResponse response = executeAndCapture(relNode);
     String dump = dumpResponse(response);
@@ -170,7 +197,7 @@ void executeRelNode_emptyResults() {
   void executeRelNode_nullValues() {
     RelNode relNode = mockRelNode("name", SqlTypeName.VARCHAR, "age", SqlTypeName.INTEGER);
     Iterable<Object[]> rows = Collections.singletonList(new Object[] {null, null});
-    when(mockExecutor.execute(relNode, null)).thenReturn(rows);
+    stubExecutorWith(relNode, rows);
 
     QueryResponse response = executeAndCapture(relNode);
     String dump = dumpResponse(response);
@@ -187,7 +214,7 @@ void executeRelNode_nullValues() {
   @Test
   void executeRelNode_errorPropagation() {
     RelNode relNode = mockRelNode("id", SqlTypeName.INTEGER);
-    when(mockExecutor.execute(relNode, null)).thenThrow(new RuntimeException("Engine failure"));
+    stubExecutorWithError(relNode, new RuntimeException("Engine failure"));
 
     Exception error = executeAndCaptureError(relNode);
     System.out.println(dumpError("executeRelNode_errorPropagation", error));

direct-query/src/test/java/org/opensearch/sql/directquery/rest/RestDirectQueryManagementActionTest.java

@@ -103,16 +103,16 @@ public void testWhenDataSourcesAreEnabled() {
         "{\"query\":\"up\",\"language\":\"promql\",\"options\":{\"queryType\":\"instant\",\"time\":\"1609459200\"}}";
     when(request.contentParser())
         .thenReturn(
-            new org.opensearch.common.xcontent.json.JsonXContentParser(
+            org.opensearch.common.xcontent.json.JsonXContent.jsonXContent.createParser(
                 org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY,
                 org.opensearch.core.xcontent.DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
-                new com.fasterxml.jackson.core.JsonFactory().createParser(requestContent)));
+                requestContent));
     when(request.contentParser())
         .thenReturn(
-            new org.opensearch.common.xcontent.json.JsonXContentParser(
+            org.opensearch.common.xcontent.json.JsonXContent.jsonXContent.createParser(
                 org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY,
                 org.opensearch.core.xcontent.DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
-                new com.fasterxml.jackson.core.JsonFactory().createParser(requestContent)));
+                requestContent));
 
     unit.handleRequest(request, channel, nodeClient);
     verify(threadPool, Mockito.times(1))
@@ -389,10 +389,10 @@ private ActionListener makeRequest(String requestContent) {
     Mockito.when(request.param("dataSources")).thenReturn("testDataSource");
     Mockito.when(request.contentParser())
         .thenReturn(
-            new org.opensearch.common.xcontent.json.JsonXContentParser(
+            org.opensearch.common.xcontent.json.JsonXContent.jsonXContent.createParser(
                 org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY,
                 org.opensearch.core.xcontent.DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
-                new com.fasterxml.jackson.core.JsonFactory().createParser(requestContent)));
+                requestContent));
     Mockito.when(request.consumedParams()).thenReturn(java.util.Collections.emptyList());
     Mockito.when(request.params()).thenReturn(java.util.Collections.emptyMap());
 

docs/dev/opensearch-nested-field-subquery.md

@@ -53,8 +53,7 @@ GET /employee_nested/_search
   "_source": {
     "includes": [
       "name"
-    ],
-    "excludes": []
+    ]
   }
 }
 
@@ -110,8 +109,7 @@ WHERE EXISTS(SELECT *
   "_source": {
     "includes": [
       "name"
-    ],
-    "excludes": []
+    ]
   }
 }
 

docs/dev/sql-nested-function-select-clause.md

@@ -106,8 +106,7 @@ A basic nested function in the SELECT clause and output DSL pushed to OpenSearch
                                         "_source": {
                                             "includes": [
                                                 "message.info"
-                                            ],
-                                            "excludes": []
+                                            ]
                                         }
                                     }
                                 }
@@ -147,8 +146,7 @@ Example with multiple SELECT clause function calls sharing same path. These two
                                             "includes": [
                                                 "message.info",
                                                 "message.author"
-                                            ],
-                                            "excludes": []
+                                            ]
                                         }
                                     }
                                 }
@@ -187,8 +185,7 @@ An example with multiple nested function calls in the SELECT clause having diffe
                                         "_source": {
                                             "includes": [
                                                 "comment.data"
-                                            ],
-                                            "excludes": []
+                                            ]
                                         }
                                     }
                                 }
@@ -207,8 +204,7 @@ An example with multiple nested function calls in the SELECT clause having diffe
                                         "_source": {
                                             "includes": [
                                                 "message.info"
-                                            ],
-                                            "excludes": []
+                                            ]
                                         }
                                     }
                                 }

docs/user/beyond/fulltext.rst

@@ -80,8 +80,7 @@ Explain::
 	    "includes" : [
 	      "account_number",
 	      "address"
-	    ],
-	    "excludes" : [ ]
+	    ]
 	  }
 	}
 
@@ -150,8 +149,7 @@ Explain::
 	    "includes" : [
 	      "account_number",
 	      "address"
-	    ],
-	    "excludes" : [ ]
+	    ]
 	  }
 	}
 
@@ -230,8 +228,7 @@ Explain::
 	    "includes" : [
 	      "firstname",
 	      "lastname"
-	    ],
-	    "excludes" : [ ]
+	    ]
 	  }
 	}
 
@@ -311,8 +308,7 @@ Explain::
 	    "includes" : [
 	      "account_number",
 	      "address"
-	    ],
-	    "excludes" : [ ]
+	    ]
 	  }
 	}
 
@@ -386,8 +382,7 @@ Explain::
 	    "includes" : [
 	      "account_number",
 	      "address"
-	    ],
-	    "excludes" : [ ]
+	    ]
 	  }
 	}
 
@@ -492,8 +487,7 @@ Explain::
 	      "account_number",
 	      "address",
 	      "_score"
-	    ],
-	    "excludes" : [ ]
+	    ]
 	  },
 	  "sort" : [
 	    {

docs/user/beyond/partiql.rst

@@ -286,8 +286,7 @@ Explain::
 	                    "_source" : {
 	                      "includes" : [
 	                        "projects.name"
-	                      ],
-	                      "excludes" : [ ]
+	                      ]
 	                    }
 	                  }
 	                }
@@ -305,8 +304,7 @@ Explain::
 	  "_source" : {
 	    "includes" : [
 	      "name"
-	    ],
-	    "excludes" : [ ]
+	    ]
 	  }
 	}
 
@@ -423,8 +421,7 @@ Explain::
 	  "_source" : {
 	    "includes" : [
 	      "name"
-	    ],
-	    "excludes" : [ ]
+	    ]
 	  }
 	}