diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java new file mode 100644 index 00000000000..0b15a36d6d8 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteAliasFieldAggregationIT.java @@ -0,0 +1,158 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; +import static org.opensearch.sql.util.MatcherUtils.verifySchemaInOrder; + +import java.io.IOException; +import java.util.List; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Integration tests for aggregation functions (MIN, MAX, FIRST, LAST, TAKE) with alias fields. + * Tests the fix for issue #4595. + */ +public class CalciteAliasFieldAggregationIT extends PPLIntegTestCase { + + private static final String TEST_INDEX_ALIAS = "test_alias_bug"; + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + createTestIndexWithAliasFields(); + } + + /** + * Create test index with alias fields mapping and insert sample data. This mirrors the + * reproduction steps from issue #4595. + */ + private void createTestIndexWithAliasFields() throws IOException { + // Delete the index if it exists (for test isolation) + try { + Request deleteIndex = new Request("DELETE", "/" + TEST_INDEX_ALIAS); + client().performRequest(deleteIndex); + } catch (ResponseException e) { + // Index doesn't exist, which is fine + } + + // Create index with alias fields + Request createIndex = new Request("PUT", "/" + TEST_INDEX_ALIAS); + createIndex.setJsonEntity( + "{\n" + + " \"mappings\": {\n" + + " \"properties\": {\n" + + " \"created_at\": {\"type\": \"date\"},\n" + + " \"@timestamp\": {\"type\": \"alias\", \"path\": \"created_at\"},\n" + + " \"value\": {\"type\": \"integer\"},\n" + + " \"value_alias\": {\"type\": \"alias\", \"path\": \"value\"}\n" + + " }\n" + + " }\n" + + "}"); + client().performRequest(createIndex); + + // Insert test documents + Request bulkRequest = new Request("POST", "/" + TEST_INDEX_ALIAS + "/_bulk?refresh=true"); + bulkRequest.setJsonEntity( + "{\"index\":{}}\n" + + "{\"created_at\": \"2024-01-01T10:00:00Z\", \"value\": 100}\n" + + "{\"index\":{}}\n" + + "{\"created_at\": \"2024-01-02T10:00:00Z\", \"value\": 200}\n" + + "{\"index\":{}}\n" + + "{\"created_at\": \"2024-01-03T10:00:00Z\", \"value\": 300}\n"); + client().performRequest(bulkRequest); + } + + @Test + public void testMinWithDateAliasField() throws IOException { + JSONObject actual = + executeQuery(String.format("source=%s | stats MIN(@timestamp)", TEST_INDEX_ALIAS)); + verifySchema(actual, schema("MIN(@timestamp)", "timestamp")); + verifyDataRows(actual, rows("2024-01-01 10:00:00")); + } + + @Test + public void testMaxWithDateAliasField() throws IOException { + JSONObject actual = + executeQuery(String.format("source=%s | stats MAX(@timestamp)", TEST_INDEX_ALIAS)); + verifySchema(actual, schema("MAX(@timestamp)", "timestamp")); + verifyDataRows(actual, rows("2024-01-03 10:00:00")); + } + + @Test + public void testMinMaxWithNumericAliasField() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | stats MIN(value_alias), MAX(value_alias)", TEST_INDEX_ALIAS)); + verifySchemaInOrder( + actual, schema("MIN(value_alias)", "int"), schema("MAX(value_alias)", "int")); + verifyDataRows(actual, rows(100, 300)); + } + + @Test + public void testFirstWithAliasField() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | sort @timestamp | stats FIRST(@timestamp)", TEST_INDEX_ALIAS)); + verifySchema(actual, schema("FIRST(@timestamp)", "timestamp")); + verifyDataRows(actual, rows("2024-01-01 10:00:00")); + } + + @Test + public void testLastWithAliasField() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | sort @timestamp | stats LAST(@timestamp)", TEST_INDEX_ALIAS)); + verifySchema(actual, schema("LAST(@timestamp)", "timestamp")); + verifyDataRows(actual, rows("2024-01-03 10:00:00")); + } + + @Test + public void testTakeWithAliasField() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | sort @timestamp | stats TAKE(@timestamp, 2)", TEST_INDEX_ALIAS)); + verifySchema(actual, schema("TAKE(@timestamp, 2)", "array")); + verifyDataRows(actual, rows(List.of("2024-01-01T10:00:00.000Z", "2024-01-02T10:00:00.000Z"))); + } + + @Test + public void testAggregationsWithOriginalFieldsStillWork() throws IOException { + JSONObject actual = + executeQuery( + String.format("source=%s | stats MIN(created_at), MAX(value)", TEST_INDEX_ALIAS)); + verifySchemaInOrder( + actual, schema("MIN(created_at)", "timestamp"), schema("MAX(value)", "int")); + verifyDataRows(actual, rows("2024-01-01 10:00:00", 300)); + } + + @Test + public void testUnaffectedAggregationsWithAliasFields() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | stats SUM(value_alias), AVG(value_alias), COUNT(value_alias)", + TEST_INDEX_ALIAS)); + verifySchemaInOrder( + actual, + schema("SUM(value_alias)", "bigint"), + schema("AVG(value_alias)", "double"), + schema("COUNT(value_alias)", "bigint")); + verifyDataRows(actual, rows(600, 200.0, 3)); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 15087d5d010..ae97cc1a666 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -641,10 +641,10 @@ public void testExplainRegexMatchInEvalWithOutScriptPushdown() throws IOExceptio // Only for Calcite @Test public void testExplainOnEarliestLatest() throws IOException { - String expected = loadExpectedPlan("explain_earliest_latest.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_earliest_latest.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( "source=%s | stats earliest(message) as earliest_message, latest(message) as" + " latest_message by server", @@ -654,10 +654,10 @@ public void testExplainOnEarliestLatest() throws IOException { // Only for Calcite @Test public void testExplainOnEarliestLatestWithCustomTimeField() throws IOException { - String expected = loadExpectedPlan("explain_earliest_latest_custom_time.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_earliest_latest_custom_time.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( "source=%s | stats earliest(message, created_at) as earliest_message," + " latest(message, created_at) as latest_message by level", @@ -667,10 +667,10 @@ public void testExplainOnEarliestLatestWithCustomTimeField() throws IOException // Only for Calcite @Test public void testExplainOnFirstLast() throws IOException { - String expected = loadExpectedPlan("explain_first_last.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_first_last.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( "source=%s | stats first(firstname) as first_name, last(firstname) as" + " last_name by gender", @@ -866,18 +866,18 @@ public void testPushdownLimitIntoAggregation() throws IOException { @Test public void testExplainMaxOnStringField() throws IOException { - String expected = loadExpectedPlan("explain_max_string_field.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_max_string_field.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString("source=opensearch-sql_test_index_account | stats max(firstname)")); + explainQueryYaml("source=opensearch-sql_test_index_account | stats max(firstname)")); } @Test public void testExplainMinOnStringField() throws IOException { - String expected = loadExpectedPlan("explain_min_string_field.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_min_string_field.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString("source=opensearch-sql_test_index_account | stats min(firstname)")); + explainQueryYaml("source=opensearch-sql_test_index_account | stats min(firstname)")); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java index 33ec09765d9..ed8c0c1d12a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java @@ -608,10 +608,10 @@ public void testDifferentFilterScriptPushDownBehaviorExplain() throws Exception @Test public void testExplainOnTake() throws IOException { - String expected = loadExpectedPlan("explain_take.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_take.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( "source=opensearch-sql_test_index_account | stats take(firstname, 2) as take")); } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.json deleted file mode 100644 index 6afa7be030c..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(earliest_message=[$1], latest_message=[$2], server=[$0])\n LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)])\n LogicalProject(server=[$1], message=[$3], @timestamp=[$2])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},earliest_message=ARG_MIN($1, $2),latest_message=ARG_MAX($1, $2)), PROJECT->[earliest_message, latest_message, server], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"server\":{\"terms\":{\"field\":\"server\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"earliest_message\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"message\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"asc\"}}]}},\"latest_message\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"message\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"desc\"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.yaml new file mode 100644 index 00000000000..3dbd4ff8c20 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(earliest_message=[$1], latest_message=[$2], server=[$0]) + LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)]) + LogicalProject(server=[$1], message=[$3], @timestamp=[$2]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},earliest_message=ARG_MIN($1, $2),latest_message=ARG_MAX($1, $2)), PROJECT->[earliest_message, latest_message, server], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"server":{"terms":{"field":"server","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"earliest_message":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"message.keyword"}],"sort":[{"@timestamp":{"order":"asc"}}]}},"latest_message":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"message.keyword"}],"sort":[{"@timestamp":{"order":"desc"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.json deleted file mode 100644 index 98d5277e419..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(earliest_message=[$1], latest_message=[$2], level=[$0])\n LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)])\n LogicalProject(level=[$4], message=[$3], created_at=[$0])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},earliest_message=ARG_MIN($1, $2),latest_message=ARG_MAX($1, $2)), PROJECT->[earliest_message, latest_message, level], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"level\":{\"terms\":{\"field\":\"level\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"earliest_message\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"message\"],\"excludes\":[]},\"sort\":[{\"created_at\":{\"order\":\"asc\"}}]}},\"latest_message\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"message\"],\"excludes\":[]},\"sort\":[{\"created_at\":{\"order\":\"desc\"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.yaml new file mode 100644 index 00000000000..4dc24423d9d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(earliest_message=[$1], latest_message=[$2], level=[$0]) + LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)]) + LogicalProject(level=[$4], message=[$3], created_at=[$0]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},earliest_message=ARG_MIN($1, $2),latest_message=ARG_MAX($1, $2)), PROJECT->[earliest_message, latest_message, level], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"level":{"terms":{"field":"level","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"earliest_message":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"message.keyword"}],"sort":[{"created_at":{"order":"asc"}}]}},"latest_message":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"message.keyword"}],"sort":[{"created_at":{"order":"desc"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.json deleted file mode 100644 index d10259db2ef..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(first_name=[$1], last_name=[$2], gender=[$0])\n LogicalAggregate(group=[{0}], first_name=[FIRST($1)], last_name=[LAST($1)])\n LogicalProject(gender=[$4], firstname=[$1])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},first_name=FIRST($1),last_name=LAST($1)), PROJECT->[first_name, last_name, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"first_name\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]}}},\"last_name\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"desc\"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.yaml new file mode 100644 index 00000000000..30f505a24a7 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(first_name=[$1], last_name=[$2], gender=[$0]) + LogicalAggregate(group=[{0}], first_name=[FIRST($1)], last_name=[LAST($1)]) + LogicalProject(gender=[$4], firstname=[$1]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},first_name=FIRST($1),last_name=LAST($1)), PROJECT->[first_name, last_name, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"first_name":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"firstname"}]}},"last_name":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"firstname"}],"sort":[{"_doc":{"order":"desc"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_max_string_field.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_max_string_field.json deleted file mode 100644 index 961cdc4687f..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_max_string_field.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], max(firstname)=[MAX($0)])\n LogicalProject(firstname=[$1])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},max(firstname)=MAX($0))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"max(firstname)\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]},\"sort\":[{\"firstname.keyword\":{\"order\":\"desc\"}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_max_string_field.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_max_string_field.yaml new file mode 100644 index 00000000000..9f44afdf68d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_max_string_field.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], max(firstname)=[MAX($0)]) + LogicalProject(firstname=[$1]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},max(firstname)=MAX($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"max(firstname)":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"firstname.keyword"}],"sort":[{"firstname.keyword":{"order":"desc"}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_min_string_field.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_min_string_field.json deleted file mode 100644 index 41a14f5e84e..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_min_string_field.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], min(firstname)=[MIN($0)])\n LogicalProject(firstname=[$1])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},min(firstname)=MIN($0))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"min(firstname)\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]},\"sort\":[{\"firstname.keyword\":{\"order\":\"asc\"}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_min_string_field.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_min_string_field.yaml new file mode 100644 index 00000000000..20bfd6b4dd7 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_min_string_field.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], min(firstname)=[MIN($0)]) + LogicalProject(firstname=[$1]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},min(firstname)=MIN($0))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"min(firstname)":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"firstname.keyword"}],"sort":[{"firstname.keyword":{"order":"asc"}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml index cee972c7b32..16c87b3ae02 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..2=[{inputs}], expr#3=[PATTERN_PARSER($t0, $t2)], expr#4=['pattern'], expr#5=[ITEM($t3, $t4)], expr#6=[SAFE_CAST($t5)], expr#7=['tokens'], expr#8=[ITEM($t3, $t7)], expr#9=[SAFE_CAST($t8)], patterns_field=[$t6], pattern_count=[$t1], tokens=[$t9], sample_logs=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},pattern_count=COUNT($1),sample_logs=TAKE($0, $2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"patterns_field":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlHsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IGZhbHNlCn10AARleHBydAYZewogICJvcCI6IHsKICAgICJuYW1lIjogIkNBU0UiLAogICAgImtpbmQiOiAiQ0FTRSIsCiAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJTRUFSQ0giLAogICAgICAgICJraW5kIjogIlNFQVJDSCIsCiAgICAgICAgInN5bnRheCI6ICJJTlRFUk5BTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogewogICAgICAgICAgICAicmFuZ2VTZXQiOiBbCiAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgInNpbmdsZXRvbiIsCiAgICAgICAgICAgICAgICAie1widmFsdWVcIjpcIlwiLFwiY2hhcnNldE5hbWVcIjpcIklTTy04ODU5LTFcIixcImNvbGxhdGlvblwiOntcImNvbGxhdGlvbk5hbWVcIjpcIklTTy04ODU5LTEkZW5fVVMkcHJpbWFyeVwiLFwiY29lcmNpYmlsaXR5XCI6XCJJTVBMSUNJVFwiLFwibG9jYWxlXCI6XCJlbl9VU1wifSxcImNoYXJzZXRcIjpcIklTTy04ODU5LTFcIixcInZhbHVlQnl0ZXNcIjpudWxsfSIKICAgICAgICAgICAgICBdCiAgICAgICAgICAgIF0sCiAgICAgICAgICAgICJudWxsQXMiOiAiVFJVRSIKICAgICAgICAgIH0sCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIiIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiUkVHRVhQX1JFUExBQ0UiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiW2EtekEtWjAtOV0rIiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiPCo+IiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQABWVtYWlsc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4ADH5xAH4AEHQABlNUUklOR35xAH4AFHQAB0tleXdvcmRxAH4AGXh4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"pattern_count":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlHsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IGZhbHNlCn10AARleHBydAYZewogICJvcCI6IHsKICAgICJuYW1lIjogIkNBU0UiLAogICAgImtpbmQiOiAiQ0FTRSIsCiAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJTRUFSQ0giLAogICAgICAgICJraW5kIjogIlNFQVJDSCIsCiAgICAgICAgInN5bnRheCI6ICJJTlRFUk5BTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogewogICAgICAgICAgICAicmFuZ2VTZXQiOiBbCiAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgInNpbmdsZXRvbiIsCiAgICAgICAgICAgICAgICAie1widmFsdWVcIjpcIlwiLFwiY2hhcnNldE5hbWVcIjpcIklTTy04ODU5LTFcIixcImNvbGxhdGlvblwiOntcImNvbGxhdGlvbk5hbWVcIjpcIklTTy04ODU5LTEkZW5fVVMkcHJpbWFyeVwiLFwiY29lcmNpYmlsaXR5XCI6XCJJTVBMSUNJVFwiLFwibG9jYWxlXCI6XCJlbl9VU1wifSxcImNoYXJzZXRcIjpcIklTTy04ODU5LTFcIixcInZhbHVlQnl0ZXNcIjpudWxsfSIKICAgICAgICAgICAgICBdCiAgICAgICAgICAgIF0sCiAgICAgICAgICAgICJudWxsQXMiOiAiVFJVRSIKICAgICAgICAgIH0sCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIiIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiUkVHRVhQX1JFUExBQ0UiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiW2EtekEtWjAtOV0rIiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiPCo+IiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQABWVtYWlsc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4ADH5xAH4AEHQABlNUUklOR35xAH4AFHQAB0tleXdvcmRxAH4AGXh4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}},"sample_logs":{"top_hits":{"from":0,"size":10,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["email"],"excludes":[]}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},pattern_count=COUNT($1),sample_logs=TAKE($0, $2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"patterns_field":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlHsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IGZhbHNlCn10AARleHBydAYZewogICJvcCI6IHsKICAgICJuYW1lIjogIkNBU0UiLAogICAgImtpbmQiOiAiQ0FTRSIsCiAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJTRUFSQ0giLAogICAgICAgICJraW5kIjogIlNFQVJDSCIsCiAgICAgICAgInN5bnRheCI6ICJJTlRFUk5BTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogewogICAgICAgICAgICAicmFuZ2VTZXQiOiBbCiAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgInNpbmdsZXRvbiIsCiAgICAgICAgICAgICAgICAie1widmFsdWVcIjpcIlwiLFwiY2hhcnNldE5hbWVcIjpcIklTTy04ODU5LTFcIixcImNvbGxhdGlvblwiOntcImNvbGxhdGlvbk5hbWVcIjpcIklTTy04ODU5LTEkZW5fVVMkcHJpbWFyeVwiLFwiY29lcmNpYmlsaXR5XCI6XCJJTVBMSUNJVFwiLFwibG9jYWxlXCI6XCJlbl9VU1wifSxcImNoYXJzZXRcIjpcIklTTy04ODU5LTFcIixcInZhbHVlQnl0ZXNcIjpudWxsfSIKICAgICAgICAgICAgICBdCiAgICAgICAgICAgIF0sCiAgICAgICAgICAgICJudWxsQXMiOiAiVFJVRSIKICAgICAgICAgIH0sCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIiIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiUkVHRVhQX1JFUExBQ0UiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiW2EtekEtWjAtOV0rIiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiPCo+IiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQABWVtYWlsc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4ADH5xAH4AEHQABlNUUklOR35xAH4AFHQAB0tleXdvcmRxAH4AGXh4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"pattern_count":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlHsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IGZhbHNlCn10AARleHBydAYZewogICJvcCI6IHsKICAgICJuYW1lIjogIkNBU0UiLAogICAgImtpbmQiOiAiQ0FTRSIsCiAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJTRUFSQ0giLAogICAgICAgICJraW5kIjogIlNFQVJDSCIsCiAgICAgICAgInN5bnRheCI6ICJJTlRFUk5BTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogewogICAgICAgICAgICAicmFuZ2VTZXQiOiBbCiAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgInNpbmdsZXRvbiIsCiAgICAgICAgICAgICAgICAie1widmFsdWVcIjpcIlwiLFwiY2hhcnNldE5hbWVcIjpcIklTTy04ODU5LTFcIixcImNvbGxhdGlvblwiOntcImNvbGxhdGlvbk5hbWVcIjpcIklTTy04ODU5LTEkZW5fVVMkcHJpbWFyeVwiLFwiY29lcmNpYmlsaXR5XCI6XCJJTVBMSUNJVFwiLFwibG9jYWxlXCI6XCJlbl9VU1wifSxcImNoYXJzZXRcIjpcIklTTy04ODU5LTFcIixcInZhbHVlQnl0ZXNcIjpudWxsfSIKICAgICAgICAgICAgICBdCiAgICAgICAgICAgIF0sCiAgICAgICAgICAgICJudWxsQXMiOiAiVFJVRSIKICAgICAgICAgIH0sCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIiIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiUkVHRVhQX1JFUExBQ0UiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiW2EtekEtWjAtOV0rIiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiPCo+IiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQABWVtYWlsc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4ADH5xAH4AEHQABlNUUklOR35xAH4AFHQAB0tleXdvcmRxAH4AGXh4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}},"sample_logs":{"top_hits":{"from":0,"size":10,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"email.keyword"}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_take.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_take.json deleted file mode 100644 index a48633f7ea2..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_take.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], take=[TAKE($0, $1)])\n LogicalProject(firstname=[$1], $f1=[2])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},take=TAKE($0, $1))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"take\":{\"top_hits\":{\"from\":0,\"size\":2,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_take.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_take.yaml new file mode 100644 index 00000000000..e395f0e7485 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_take.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], take=[TAKE($0, $1)]) + LogicalProject(firstname=[$1], $f1=[2]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},take=TAKE($0, $1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"take":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"firstname.keyword"}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest.json deleted file mode 100644 index ad2df524707..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(earliest_message=[$1], latest_message=[$2], server=[$0])\n LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)])\n LogicalProject(server=[$1], message=[$3], @timestamp=[$2])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], earliest_message=[$t1], latest_message=[$t2], server=[$t0])\n EnumerableAggregate(group=[{1}], earliest_message=[ARG_MIN($3, $2)], latest_message=[ARG_MAX($3, $2)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest.yaml new file mode 100644 index 00000000000..32496a5b10e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(earliest_message=[$1], latest_message=[$2], server=[$0]) + LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)]) + LogicalProject(server=[$1], message=[$3], @timestamp=[$2]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], earliest_message=[$t1], latest_message=[$t2], server=[$t0]) + EnumerableAggregate(group=[{1}], earliest_message=[ARG_MIN($3, $2)], latest_message=[ARG_MAX($3, $2)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest_custom_time.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest_custom_time.json deleted file mode 100644 index ef5ebfe9e1c..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest_custom_time.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(earliest_message=[$1], latest_message=[$2], level=[$0])\n LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)])\n LogicalProject(level=[$4], message=[$3], created_at=[$0])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], earliest_message=[$t1], latest_message=[$t2], level=[$t0])\n EnumerableAggregate(group=[{4}], earliest_message=[ARG_MIN($3, $0)], latest_message=[ARG_MAX($3, $0)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest_custom_time.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest_custom_time.yaml new file mode 100644 index 00000000000..d9e4320bc4a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_earliest_latest_custom_time.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(earliest_message=[$1], latest_message=[$2], level=[$0]) + LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)]) + LogicalProject(level=[$4], message=[$3], created_at=[$0]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], earliest_message=[$t1], latest_message=[$t2], level=[$t0]) + EnumerableAggregate(group=[{4}], earliest_message=[ARG_MIN($3, $0)], latest_message=[ARG_MAX($3, $0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_first_last.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_first_last.json deleted file mode 100644 index cd87821aed6..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_first_last.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(first_name=[$1], last_name=[$2], gender=[$0])\n LogicalAggregate(group=[{0}], first_name=[FIRST($1)], last_name=[LAST($1)])\n LogicalProject(gender=[$4], firstname=[$1])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], first_name=[$t1], last_name=[$t2], gender=[$t0])\n EnumerableAggregate(group=[{4}], first_name=[FIRST($1)], last_name=[LAST($1)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n"}} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_first_last.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_first_last.yaml new file mode 100644 index 00000000000..ee569fbbf3a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_first_last.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(first_name=[$1], last_name=[$2], gender=[$0]) + LogicalAggregate(group=[{0}], first_name=[FIRST($1)], last_name=[LAST($1)]) + LogicalProject(gender=[$4], firstname=[$1]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], first_name=[$t1], last_name=[$t2], gender=[$t0]) + EnumerableAggregate(group=[{4}], first_name=[FIRST($1)], last_name=[LAST($1)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_max_string_field.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_max_string_field.json deleted file mode 100644 index 8a84763e33c..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_max_string_field.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], max(firstname)=[MAX($0)])\n LogicalProject(firstname=[$1])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableAggregate(group=[{}], max(firstname)=[MAX($1)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_max_string_field.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_max_string_field.yaml new file mode 100644 index 00000000000..fe8927692ab --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_max_string_field.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], max(firstname)=[MAX($0)]) + LogicalProject(firstname=[$1]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableAggregate(group=[{}], max(firstname)=[MAX($1)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_min_string_field.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_min_string_field.json deleted file mode 100644 index 320b519c442..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_min_string_field.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], min(firstname)=[MIN($0)])\n LogicalProject(firstname=[$1])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableAggregate(group=[{}], min(firstname)=[MIN($1)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_min_string_field.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_min_string_field.yaml new file mode 100644 index 00000000000..027ecd556d9 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_min_string_field.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], min(firstname)=[MIN($0)]) + LogicalProject(firstname=[$1]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableAggregate(group=[{}], min(firstname)=[MIN($1)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_take.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_take.json deleted file mode 100644 index cac543209cc..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_take.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], take=[TAKE($0, $1)])\n LogicalProject(firstname=[$1], $f1=[2])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableAggregate(group=[{}], take=[TAKE($0, $1)])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[2], firstname=[$t1], $f1=[$t17])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_take.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_take.yaml new file mode 100644 index 00000000000..0f98cbf8ab5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_take.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], take=[TAKE($0, $1)]) + LogicalProject(firstname=[$1], $f1=[2]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableAggregate(group=[{}], take=[TAKE($0, $1)]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[2], firstname=[$t1], $f1=[$t17]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.json deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.yaml index 7a2395fca4b..15f29f544ae 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.yaml @@ -14,7 +14,7 @@ root: missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"pattern_count\"\ :{\"value_count\":{\"field\":\"_index\"}},\"sample_logs\":{\"top_hits\"\ :{\"from\":0,\"size\":10,\"version\":false,\"seq_no_primary_term\":false,\"\ - explain\":false,\"_source\":{\"includes\":[\"email\"],\"excludes\":[]}}}}}}},\ - \ needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + explain\":false,\"fields\":[{\"field\":\"email\"}]}}}}}}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ \ searchResponse=null)" children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_take.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_take.json deleted file mode 100644 index 5e623b3fd26..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_take.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[take]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"take\":{\"top_hits\":{\"from\":0,\"size\":2,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_take.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_take.yaml new file mode 100644 index 00000000000..7e4de15c4d4 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_take.yaml @@ -0,0 +1,14 @@ +root: + name: ProjectOperator + description: + fields: "[take]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\"\ + :{\"take\":{\"top_hits\":{\"from\":0,\"size\":2,\"version\":false,\"seq_no_primary_term\"\ + :false,\"explain\":false,\"fields\":[{\"field\":\"firstname\"}]}}}}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] \ No newline at end of file diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java index 82ee2d84520..e2bd4ca4e30 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java @@ -417,7 +417,7 @@ private static Pair createRegularAggregation( } else { yield Pair.of( AggregationBuilders.topHits(aggFieldName) - .fetchSource(helper.inferNamedField(args.getFirst()).getRootName(), null) + .fetchField(helper.inferNamedField(args.getFirst()).getReferenceForTermQuery()) .size(1) .from(0) .sort( @@ -436,7 +436,7 @@ private static Pair createRegularAggregation( } else { yield Pair.of( AggregationBuilders.topHits(aggFieldName) - .fetchSource(helper.inferNamedField(args.getFirst()).getRootName(), null) + .fetchField(helper.inferNamedField(args.getFirst()).getReferenceForTermQuery()) .size(1) .from(0) .sort( @@ -459,20 +459,20 @@ private static Pair createRegularAggregation( new StatsParser(ExtendedStats::getStdDeviationPopulation, aggFieldName)); case ARG_MAX -> Pair.of( AggregationBuilders.topHits(aggFieldName) - .fetchSource(helper.inferNamedField(args.getFirst()).getRootName(), null) + .fetchField(helper.inferNamedField(args.getFirst()).getReferenceForTermQuery()) .size(1) .from(0) .sort( - helper.inferNamedField(args.get(1)).getRootName(), + helper.inferNamedField(args.get(1)).getReferenceForTermQuery(), org.opensearch.search.sort.SortOrder.DESC), new ArgMaxMinParser(aggFieldName)); case ARG_MIN -> Pair.of( AggregationBuilders.topHits(aggFieldName) - .fetchSource(helper.inferNamedField(args.getFirst()).getRootName(), null) + .fetchField(helper.inferNamedField(args.getFirst()).getReferenceForTermQuery()) .size(1) .from(0) .sort( - helper.inferNamedField(args.get(1)).getRootName(), + helper.inferNamedField(args.get(1)).getReferenceForTermQuery(), org.opensearch.search.sort.SortOrder.ASC), new ArgMaxMinParser(aggFieldName)); case OTHER_FUNCTION -> { @@ -481,7 +481,7 @@ private static Pair createRegularAggregation( yield switch (functionName) { case TAKE -> Pair.of( AggregationBuilders.topHits(aggFieldName) - .fetchSource(helper.inferNamedField(args.getFirst()).getRootName(), null) + .fetchField(helper.inferNamedField(args.getFirst()).getReferenceForTermQuery()) .size(helper.inferValue(args.getLast(), Integer.class)) .from(0), new TopHitsParser(aggFieldName)); @@ -489,7 +489,8 @@ yield switch (functionName) { TopHitsAggregationBuilder firstBuilder = AggregationBuilders.topHits(aggFieldName).size(1).from(0); if (!args.isEmpty()) { - firstBuilder.fetchSource(helper.inferNamedField(args.getFirst()).getRootName(), null); + firstBuilder.fetchField( + helper.inferNamedField(args.getFirst()).getReferenceForTermQuery()); } yield Pair.of(firstBuilder, new TopHitsParser(aggFieldName, true)); } @@ -500,7 +501,8 @@ yield switch (functionName) { .from(0) .sort("_doc", org.opensearch.search.sort.SortOrder.DESC); if (!args.isEmpty()) { - lastBuilder.fetchSource(helper.inferNamedField(args.getFirst()).getRootName(), null); + lastBuilder.fetchField( + helper.inferNamedField(args.getFirst()).getReferenceForTermQuery()); } yield Pair.of(lastBuilder, new TopHitsParser(aggFieldName, true)); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/ArgMaxMinParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/ArgMaxMinParser.java index 2ff1e511713..55dacd7081c 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/ArgMaxMinParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/ArgMaxMinParser.java @@ -27,13 +27,12 @@ public Map parse(Aggregation agg) { return Collections.singletonMap(agg.getName(), null); } - Map source = hits[0].getSourceAsMap(); - - if (source.isEmpty()) { - return Collections.singletonMap(agg.getName(), null); - } else { - Object value = source.values().iterator().next(); + // Get value from fields (fetchField) + if (hits[0].getFields() != null && !hits[0].getFields().isEmpty()) { + Object value = hits[0].getFields().values().iterator().next().getValue(); return Collections.singletonMap(agg.getName(), value); } + + return Collections.singletonMap(agg.getName(), null); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/TopHitsParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/TopHitsParser.java index 9179670d1a7..c9d9cf61d9e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/TopHitsParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/TopHitsParser.java @@ -42,21 +42,24 @@ public Map parse(Aggregation agg) { } if (returnSingleValue) { - // Extract the single value from the first (and only) hit - Map source = hits[0].getSourceAsMap(); - if (source.isEmpty()) { - return Collections.singletonMap(agg.getName(), null); + // Extract the single value from the first (and only) hit from fields (fetchField) + if (hits[0].getFields() != null && !hits[0].getFields().isEmpty()) { + Object value = hits[0].getFields().values().iterator().next().getValue(); + return Collections.singletonMap(agg.getName(), value); } - // Get the first value from the source map - Object value = source.values().iterator().next(); - return Collections.singletonMap(agg.getName(), value); + return Collections.singletonMap(agg.getName(), null); } else { - // Return all values as a list - return Collections.singletonMap( - agg.getName(), - Arrays.stream(hits) - .flatMap(h -> h.getSourceAsMap().values().stream()) - .collect(Collectors.toList())); + // Return all values as a list from fields (fetchField) + if (hits[0].getFields() != null && !hits[0].getFields().isEmpty()) { + return Collections.singletonMap( + agg.getName(), + Arrays.stream(hits) + .flatMap(h -> h.getFields().values().stream()) + .map(f -> f.getValue()) + .filter(v -> v != null) // Filter out null values + .collect(Collectors.toList())); + } + return Collections.singletonMap(agg.getName(), Collections.emptyList()); } } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java index 9cb3f9824b8..0f523d65341 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilder.java @@ -215,7 +215,7 @@ private Pair make( String name, MetricParser parser) { String fieldName = ((ReferenceExpression) expression).getAttr(); - builder.fetchSource(fieldName, null); + builder.fetchField(fieldName); builder.size(size.valueOf().integerValue()); builder.from(0); if (condition != null) { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java index 57f7c4ea044..5dc88ad5d64 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchAggregationResponseParserTest.java @@ -276,16 +276,16 @@ void top_hits_aggregation_should_pass() { + " \"_index\": \"accounts\",\n" + " \"_id\": \"1\",\n" + " \"_score\": 1.0,\n" - + " \"_source\": {\n" - + " \"gender\": \"m\"\n" + + " \"fields\": {\n" + + " \"gender\": [\"m\"]\n" + " }\n" + " },\n" + " {\n" + " \"_index\": \"accounts\",\n" + " \"_id\": \"2\",\n" + " \"_score\": 1.0,\n" - + " \"_source\": {\n" - + " \"gender\": \"f\"\n" + + " \"fields\": {\n" + + " \"gender\": [\"f\"]\n" + " }\n" + " }\n" + " ]\n" diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java index 64ae7b187c2..eeacbecd7aa 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/MetricAggregationBuilderTest.java @@ -409,10 +409,9 @@ void should_build_top_hits_aggregation() { + " \"version\" : false,%n" + " \"seq_no_primary_term\" : false,%n" + " \"explain\" : false,%n" - + " \"_source\" : {%n" - + " \"includes\" : [ \"name\" ],%n" - + " \"excludes\" : [ ]%n" - + " }%n" + + " \"fields\" : [ {%n" + + " \"field\" : \"name\"%n" + + " } ]%n" + " }%n" + " }%n" + "}"), @@ -449,10 +448,9 @@ void should_build_filtered_top_hits_aggregation() { + " \"version\" : false,%n" + " \"seq_no_primary_term\" : false,%n" + " \"explain\" : false,%n" - + " \"_source\" : {%n" - + " \"includes\" : [ \"name\" ],%n" - + " \"excludes\" : [ ]%n" - + " }%n" + + " \"fields\" : [ {%n" + + " \"field\" : \"name\"%n" + + " } ]%n" + " }%n" + " }%n" + " }%n"