diff --git a/core/src/main/java/org/opensearch/sql/utils/YamlFormatter.java b/core/src/main/java/org/opensearch/sql/utils/YamlFormatter.java index 8df2bbb6919..3ccafb34abd 100644 --- a/core/src/main/java/org/opensearch/sql/utils/YamlFormatter.java +++ b/core/src/main/java/org/opensearch/sql/utils/YamlFormatter.java @@ -7,7 +7,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; @@ -27,7 +26,6 @@ public class YamlFormatter { YAMLGenerator.Feature.ALWAYS_QUOTE_NUMBERS_AS_STRINGS); // Quote numeric strings yamlFactory.enable(YAMLGenerator.Feature.INDENT_ARRAYS_WITH_INDICATOR); YAML_MAPPER = new ObjectMapper(yamlFactory); - YAML_MAPPER.enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); } /** diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 99ef4a8830c..fd6ffbb70fa 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -270,9 +270,9 @@ public void testExplainWithTimechartAvg() throws IOException { var result = explainQueryToString("source=events | timechart span=1m avg(cpu_usage) by host"); String expected = !isPushdownDisabled() - ? loadFromFile("expectedOutput/calcite/explain_timechart.json") - : loadFromFile("expectedOutput/calcite/explain_timechart_no_pushdown.json"); - assertJsonEqualsIgnoreId(expected, result); + ? loadFromFile("expectedOutput/calcite/explain_timechart.yaml") + : loadFromFile("expectedOutput/calcite/explain_timechart_no_pushdown.yaml"); + assertYamlEqualsJsonIgnoreId(expected, result); } @Test @@ -280,9 +280,9 @@ public void testExplainWithTimechartCount() throws IOException { var result = explainQueryToString("source=events | timechart span=1m count() by host"); String expected = !isPushdownDisabled() - ? loadFromFile("expectedOutput/calcite/explain_timechart_count.json") - : loadFromFile("expectedOutput/calcite/explain_timechart_count_no_pushdown.json"); - assertJsonEqualsIgnoreId(expected, result); + ? loadFromFile("expectedOutput/calcite/explain_timechart_count.yaml") + : loadFromFile("expectedOutput/calcite/explain_timechart_count_no_pushdown.yaml"); + assertYamlEqualsJsonIgnoreId(expected, result); } @Test @@ -335,8 +335,8 @@ public void testExplainStatsWithBinsOnTimeField() throws IOException { @Test public void testExplainBinWithSpan() throws IOException { - String expected = loadExpectedPlan("explain_bin_span.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_bin_span.yaml"); + assertYamlEqualsJsonIgnoreId( expected, explainQueryToString( "source=opensearch-sql_test_index_account | bin age span=10 | head 5")); @@ -362,8 +362,8 @@ public void testExplainBinWithStartEnd() throws IOException { @Test public void testExplainBinWithAligntime() throws IOException { - String expected = loadExpectedPlan("explain_bin_aligntime.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_bin_aligntime.yaml"); + assertYamlEqualsJsonIgnoreId( expected, explainQueryToString( "source=opensearch-sql_test_index_time_data | bin @timestamp span=2h aligntime=latest |" @@ -413,8 +413,8 @@ public void testEventstatsDistinctCountFunctionExplain() throws IOException { // Only for Calcite, as v2 gets unstable serialized string for function @Test public void testExplainOnAggregationWithSumEnhancement() throws IOException { - String expected = loadExpectedPlan("explain_agg_with_sum_enhancement.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_agg_with_sum_enhancement.yaml"); + assertYamlEqualsJsonIgnoreId( expected, explainQueryToString( String.format( @@ -545,16 +545,16 @@ public void testRegexExplain() throws IOException { String query = "source=opensearch-sql_test_index_account | regex lastname='^[A-Z][a-z]+$' | head 5"; var result = explainQueryToString(query); - String expected = loadExpectedPlan("explain_regex.json"); - assertJsonEqualsIgnoreId(expected, result); + String expected = loadExpectedPlan("explain_regex.yaml"); + assertYamlEqualsJsonIgnoreId(expected, result); } @Test public void testRegexNegatedExplain() throws IOException { String query = "source=opensearch-sql_test_index_account | regex lastname!='.*son$' | head 5"; var result = explainQueryToString(query); - String expected = loadExpectedPlan("explain_regex_negated.json"); - assertJsonEqualsIgnoreId(expected, result); + String expected = loadExpectedPlan("explain_regex_negated.yaml"); + assertYamlEqualsJsonIgnoreId(expected, result); } @Test @@ -581,8 +581,8 @@ public void testRexExplain() throws IOException { "source=opensearch-sql_test_index_account | rex field=lastname \\\"(?^[A-Z])\\\" |" + " head 5"; var result = explainQueryToString(query); - String expected = loadExpectedPlan("explain_rex.json"); - assertJsonEqualsIgnoreId(expected, result); + String expected = loadExpectedPlan("explain_rex.yaml"); + assertYamlEqualsJsonIgnoreId(expected, result); } @Test @@ -628,8 +628,8 @@ public void testPushdownLimitIntoAggregation() throws IOException { expected, explainQueryToString("source=opensearch-sql_test_index_account | stats count() by state")); - expected = loadExpectedPlan("explain_limit_agg_pushdown2.json"); - assertJsonEqualsIgnoreId( + expected = loadExpectedPlan("explain_limit_agg_pushdown2.yaml"); + assertYamlEqualsJsonIgnoreId( expected, explainQueryToString( "source=opensearch-sql_test_index_account | stats count() by state | head 100")); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java index cd4e2f5d694..d888833c05d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLExplainIT.java @@ -16,6 +16,7 @@ public class CalcitePPLExplainIT extends PPLIntegTestCase { @Override public void init() throws Exception { + GlobalPushdownConfig.enabled = false; super.init(); enableCalcite(); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java index 13245ec8c94..981b418140e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteSortCommandIT.java @@ -39,18 +39,6 @@ public void testPushdownSortPlusExpression() throws IOException { Locale.ROOT, "source=%s | eval age2 = age + 2 | sort age2 | fields age | head 2", TEST_INDEX_BANK); - String explained = explainQueryToString(ppl); - if (!isPushdownDisabled()) { - assertTrue( - explained.contains( - "[SORT->[{\\n" - + " \\\"age\\\" : {\\n" - + " \\\"order\\\" : \\\"asc\\\",\\n" - + " \\\"missing\\\" : \\\"_first\\\"\\n" - + " }\\n" - + "}]")); - } - JSONObject result = executeQuery(ppl); verifyOrder(result, rows(28), rows(32)); } @@ -62,18 +50,6 @@ public void testPushdownSortMinusExpression() throws IOException { Locale.ROOT, "source=%s | eval age2 = 1 - age | sort age2 | fields age | head 2", TEST_INDEX_BANK); - String explained = explainQueryToString(ppl); - if (!isPushdownDisabled()) { - assertTrue( - explained.contains( - "[SORT->[{\\n" - + " \\\"age\\\" : {\\n" - + " \\\"order\\\" : \\\"desc\\\",\\n" - + " \\\"missing\\\" : \\\"_first\\\"\\n" - + " }\\n" - + "}]")); - } - JSONObject result = executeQuery(ppl); verifyOrder(result, rows(39), rows(36)); } @@ -85,18 +61,6 @@ public void testPushdownSortTimesExpression() throws IOException { Locale.ROOT, "source=%s | eval age2 = 5 * age | sort age2 | fields age | head 2", TEST_INDEX_BANK); - String explained = explainQueryToString(ppl); - if (!isPushdownDisabled()) { - assertTrue( - explained.contains( - "[SORT->[{\\n" - + " \\\"age\\\" : {\\n" - + " \\\"order\\\" : \\\"asc\\\",\\n" - + " \\\"missing\\\" : \\\"_first\\\"\\n" - + " }\\n" - + "}]")); - } - JSONObject result = executeQuery(ppl); verifyOrder(result, rows(28), rows(32)); } @@ -108,23 +72,6 @@ public void testPushdownSortByMultiExpressions() throws IOException { Locale.ROOT, "source=%s | eval age2 = 5 * age | sort gender, age2 | fields gender, age | head 2", TEST_INDEX_BANK); - String explained = explainQueryToString(ppl); - if (!isPushdownDisabled()) { - assertTrue( - explained.contains( - "[SORT->[{\\n" - + " \\\"gender.keyword\\\" : {\\n" - + " \\\"order\\\" : \\\"asc\\\",\\n" - + " \\\"missing\\\" : \\\"_first\\\"\\n" - + " }\\n" - + "}, {\\n" - + " \\\"age\\\" : {\\n" - + " \\\"order\\\" : \\\"asc\\\",\\n" - + " \\\"missing\\\" : \\\"_first\\\"\\n" - + " }\\n" - + "}]")); - } - JSONObject result = executeQuery(ppl); verifyOrder(result, rows("F", 28), rows("F", 34)); } @@ -136,18 +83,6 @@ public void testPushdownSortCastExpression() throws IOException { Locale.ROOT, "source=%s | eval age2 = cast(age * 5 as long) | sort age2 | fields age | head 2", TEST_INDEX_BANK); - String explained = explainQueryToString(ppl); - if (!isPushdownDisabled()) { - assertTrue( - explained.contains( - "[SORT->[{\\n" - + " \\\"age\\\" : {\\n" - + " \\\"order\\\" : \\\"asc\\\",\\n" - + " \\\"missing\\\" : \\\"_first\\\"\\n" - + " }\\n" - + "}]")); - } - JSONObject result = executeQuery(ppl); verifyOrder(result, rows(28), rows(32)); } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java index 3fa8a545622..89c29a551c4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java @@ -11,6 +11,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_OTEL_LOGS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOGS; import static org.opensearch.sql.util.MatcherUtils.assertJsonEqualsIgnoreId; +import static org.opensearch.sql.util.MatcherUtils.assertYamlEqualsJsonIgnoreId; import java.io.IOException; import java.util.Locale; @@ -34,8 +35,8 @@ public void init() throws Exception { @Test public void testExplain() throws IOException { - String expected = loadExpectedPlan("explain_output.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_output.yaml"); + assertYamlEqualsJsonIgnoreId( expected, explainQueryToString( "source=opensearch-sql_test_index_account" @@ -50,8 +51,8 @@ public void testExplain() throws IOException { @Test public void testFilterPushDownExplain() throws IOException { - String expected = loadExpectedPlan("explain_filter_push.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_filter_push.yaml"); + assertYamlEqualsJsonIgnoreId( expected, explainQueryToString( "source=opensearch-sql_test_index_account" @@ -63,8 +64,8 @@ public void testFilterPushDownExplain() throws IOException { @Test public void testFilterByCompareStringTimestampPushDownExplain() throws IOException { - String expected = loadExpectedPlan("explain_filter_push_compare_timestamp_string.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_filter_push_compare_timestamp_string.yaml"); + assertYamlEqualsJsonIgnoreId( expected, explainQueryToString( "source=opensearch-sql_test_index_bank" @@ -74,8 +75,8 @@ public void testFilterByCompareStringTimestampPushDownExplain() throws IOExcepti @Test public void testFilterByCompareStringDatePushDownExplain() throws IOException { - String expected = loadExpectedPlan("explain_filter_push_compare_date_string.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_filter_push_compare_date_string.yaml"); + assertYamlEqualsJsonIgnoreId( expected, explainQueryToString( "source=opensearch-sql_test_index_date_formats | fields yyyy-MM-dd" @@ -85,8 +86,8 @@ public void testFilterByCompareStringDatePushDownExplain() throws IOException { @Test public void testFilterByCompareStringTimePushDownExplain() throws IOException { - String expected = loadExpectedPlan("explain_filter_push_compare_time_string.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_filter_push_compare_time_string.yaml"); + assertYamlEqualsJsonIgnoreId( expected, explainQueryToString( "source=opensearch-sql_test_index_date_formats | fields custom_time" @@ -137,8 +138,8 @@ public void testWeekArgumentCoercion() throws IOException { @Test public void testFilterAndAggPushDownExplain() throws IOException { - String expected = loadExpectedPlan("explain_filter_agg_push.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_filter_agg_push.yaml"); + assertYamlEqualsJsonIgnoreId( expected, explainQueryToString( "source=opensearch-sql_test_index_account" @@ -168,8 +169,8 @@ public void testSortPushDownExplain() throws IOException { @Test public void testSortWithCountPushDownExplain() throws IOException { - String expected = loadExpectedPlan("explain_sort_count_push.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_sort_count_push.yaml"); + assertYamlEqualsJsonIgnoreId( expected, explainQueryToString("source=opensearch-sql_test_index_account | sort 5 age | fields age")); } @@ -252,8 +253,8 @@ public void testSortWithRenameExplain() throws IOException { */ @Test public void testSortThenLimitExplain() throws IOException { - String expected = loadExpectedPlan("explain_sort_then_limit_push.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_sort_then_limit_push.yaml"); + assertYamlEqualsJsonIgnoreId( expected, explainQueryToString( "source=opensearch-sql_test_index_account" @@ -270,8 +271,8 @@ public void testSortThenLimitExplain() throws IOException { public void testLimitThenSortExplain() throws IOException { // TODO: Fix the expected output in expectedOutput/ppl/explain_limit_then_sort_push.json (v2) // limit-then-sort should not be pushed down. - String expected = loadExpectedPlan("explain_limit_then_sort_push.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_limit_then_sort_push.yaml"); + assertYamlEqualsJsonIgnoreId( expected, explainQueryToString( "source=opensearch-sql_test_index_account" @@ -282,8 +283,8 @@ public void testLimitThenSortExplain() throws IOException { @Test public void testLimitPushDownExplain() throws IOException { - String expected = loadExpectedPlan("explain_limit_push.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_limit_push.yaml"); + assertYamlEqualsJsonIgnoreId( expected, explainQueryToString( "source=opensearch-sql_test_index_account" @@ -294,8 +295,8 @@ public void testLimitPushDownExplain() throws IOException { @Test public void testLimitWithFilterPushdownExplain() throws IOException { - String expectedFilterThenLimit = loadExpectedPlan("explain_filter_then_limit_push.json"); - assertJsonEqualsIgnoreId( + String expectedFilterThenLimit = loadExpectedPlan("explain_filter_then_limit_push.yaml"); + assertYamlEqualsJsonIgnoreId( expectedFilterThenLimit, explainQueryToString( "source=opensearch-sql_test_index_account" @@ -305,8 +306,8 @@ public void testLimitWithFilterPushdownExplain() throws IOException { // The filter in limit-then-filter queries should not be pushed since the current DSL will // execute it as filter-then-limit - String expectedLimitThenFilter = loadExpectedPlan("explain_limit_then_filter_push.json"); - assertJsonEqualsIgnoreId( + String expectedLimitThenFilter = loadExpectedPlan("explain_limit_then_filter_push.yaml"); + assertYamlEqualsJsonIgnoreId( expectedLimitThenFilter, explainQueryToString( "source=opensearch-sql_test_index_account" @@ -317,8 +318,8 @@ public void testLimitWithFilterPushdownExplain() throws IOException { @Test public void testMultipleLimitExplain() throws IOException { - String expected5Then10 = loadExpectedPlan("explain_limit_5_10_push.json"); - assertJsonEqualsIgnoreId( + String expected5Then10 = loadExpectedPlan("explain_limit_5_10_push.yaml"); + assertYamlEqualsJsonIgnoreId( expected5Then10, explainQueryToString( "source=opensearch-sql_test_index_account" @@ -326,8 +327,8 @@ public void testMultipleLimitExplain() throws IOException { + "| head 10 " + "| fields age")); - String expected10Then5 = loadExpectedPlan("explain_limit_10_5_push.json"); - assertJsonEqualsIgnoreId( + String expected10Then5 = loadExpectedPlan("explain_limit_10_5_push.yaml"); + assertYamlEqualsJsonIgnoreId( expected10Then5, explainQueryToString( "source=opensearch-sql_test_index_account" @@ -335,8 +336,8 @@ public void testMultipleLimitExplain() throws IOException { + "| head 5 " + "| fields age")); - String expected10from1then10from2 = loadExpectedPlan("explain_limit_10from1_10from2_push.json"); - assertJsonEqualsIgnoreId( + String expected10from1then10from2 = loadExpectedPlan("explain_limit_10from1_10from2_push.yaml"); + assertYamlEqualsJsonIgnoreId( expected10from1then10from2, explainQueryToString( "source=opensearch-sql_test_index_account" @@ -345,8 +346,8 @@ public void testMultipleLimitExplain() throws IOException { + "| fields age")); // The second limit should not be pushed down for limit-filter-limit queries - String expected10ThenFilterThen5 = loadExpectedPlan("explain_limit_10_filter_5_push.json"); - assertJsonEqualsIgnoreId( + String expected10ThenFilterThen5 = loadExpectedPlan("explain_limit_10_filter_5_push.yaml"); + assertYamlEqualsJsonIgnoreId( expected10ThenFilterThen5, explainQueryToString( "source=opensearch-sql_test_index_account" @@ -358,8 +359,8 @@ public void testMultipleLimitExplain() throws IOException { @Test public void testLimitWithMultipleOffsetPushdownExplain() throws IOException { - String expected = loadExpectedPlan("explain_limit_offsets_push.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_limit_offsets_push.yaml"); + assertYamlEqualsJsonIgnoreId( expected, explainQueryToString( "source=opensearch-sql_test_index_account" @@ -380,8 +381,8 @@ public void testFillNullPushDownExplain() throws IOException { @Test public void testTrendlinePushDownExplain() throws IOException { - String expected = loadExpectedPlan("explain_trendline_push.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_trendline_push.yaml"); + assertYamlEqualsJsonIgnoreId( expected, explainQueryToString( "source=opensearch-sql_test_index_account" @@ -392,9 +393,9 @@ public void testTrendlinePushDownExplain() throws IOException { @Test public void testTrendlineWithSortPushDownExplain() throws IOException { - String expected = loadExpectedPlan("explain_trendline_sort_push.json"); + String expected = loadExpectedPlan("explain_trendline_sort_push.yaml"); // Sort will not be pushed down because there's a head before it. - assertJsonEqualsIgnoreId( + assertYamlEqualsJsonIgnoreId( expected, explainQueryToString( "source=opensearch-sql_test_index_account" @@ -426,8 +427,8 @@ public void testPatternsSimplePatternMethodWithoutAggExplain() throws IOExceptio @Test public void testPatternsSimplePatternMethodWithAggPushDownExplain() throws IOException { - String expected = loadExpectedPlan("explain_patterns_simple_pattern_agg_push.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_patterns_simple_pattern_agg_push.yaml"); + assertYamlEqualsJsonIgnoreId( expected, explainQueryToString( "source=opensearch-sql_test_index_account | patterns email mode=aggregation")); diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml index ddeefad911b..722be1bec9f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml @@ -6,5 +6,5 @@ calcite: LogicalProject(gender=[$4]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], count()=[$t0], count()0=[$t0], gender=[$t1]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->10000, PROJECT->[count(), gender]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], c1=[$t1], gender=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml index 000995f3c2a..61724171f7a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml @@ -6,5 +6,5 @@ calcite: LogicalProject(gender=[$4], balance=[$3]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - EnumerableCalc(expr#0..1=[{inputs}], c1=[$t0], c10=[$t0], gender=[$t1]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c1=COUNT($1)), LIMIT->10000, PROJECT->[c1, gender]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"c1":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + EnumerableCalc(expr#0..1=[{inputs}], c1=[$t1], c2=[$t1], gender=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c1=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"c1":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_timestamp_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_timestamp_push.json index 4d0e7f7c31d..195770e8c37 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_timestamp_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_timestamp_push.json @@ -1,6 +1 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[3])\n LogicalProject(count()=[$1], t=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(t=[UNIX_TIMESTAMP($3)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT->[0 ASC FIRST], LIMIT->3, PROJECT->[count(), t], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":3,\"sources\":[{\"t\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0AWd7CiAgIm9wIjogewogICAgIm5hbWUiOiAiVU5JWF9USU1FU1RBTVAiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMywKICAgICAgIm5hbWUiOiAiJDMiCiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJET1VCTEUiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfXQACmZpZWxkVHlwZXNzcgAXamF2YS51dGlsLkxpbmtlZEhhc2hNYXA0wE5cEGzA+wIAAVoAC2FjY2Vzc09yZGVyeHIAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAAGHcIAAAAIAAAAA10AA5hY2NvdW50X251bWJlcn5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd0AAlmaXJzdG5hbWV+cQB+AAt0AAZTVFJJTkd0AAdhZGRyZXNzc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+ABR4cH5xAH4AC3QAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgAMdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AH3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AIQAAAABzcQB+AAAAAAADdwQAAAAAeHQACWJpcnRoZGF0ZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGVUeXBlni1SrhB9yq8CAAFMAAdmb3JtYXRzdAAQTGphdmEvdXRpbC9MaXN0O3hxAH4AFX5xAH4AC3QACVRJTUVTVEFNUH5xAH4AG3QABERhdGVxAH4AIHNxAH4AAAAAAAF3BAAAAAB4dAAGZ2VuZGVyc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABVxAH4AEH5xAH4AG3QAB0tleXdvcmRxAH4AIHh0AARjaXR5cQB+ABB0AAhsYXN0bmFtZXEAfgAQdAAHYmFsYW5jZXEAfgANdAAIZW1wbG95ZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABXN0YXRlc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnEAfgAxcQB+ADJ4dAADYWdlfnEAfgALdAAHSU5URUdFUnQABWVtYWlsc3EAfgATcQB+ABlxAH4AHHEAfgAgcQB+ACR0AARtYWxlfnEAfgALdAAHQk9PTEVBTngAeA==\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} +{"calcite":{"logical":"LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[3])\n LogicalProject(count()=[$1], t=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(t=[UNIX_TIMESTAMP($3)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n","physical":"CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT->[0 ASC FIRST], PROJECT->[count(), t], LIMIT->3, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":3,\"sources\":[{\"t\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0AWd7CiAgIm9wIjogewogICAgIm5hbWUiOiAiVU5JWF9USU1FU1RBTVAiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMywKICAgICAgIm5hbWUiOiAiJDMiCiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJET1VCTEUiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfXQACmZpZWxkVHlwZXNzcgAXamF2YS51dGlsLkxpbmtlZEhhc2hNYXA0wE5cEGzA+wIAAVoAC2FjY2Vzc09yZGVyeHIAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAAGHcIAAAAIAAAAA10AA5hY2NvdW50X251bWJlcn5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd0AAlmaXJzdG5hbWV+cQB+AAt0AAZTVFJJTkd0AAdhZGRyZXNzc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+ABR4cH5xAH4AC3QAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgAMdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AH3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AIQAAAABzcQB+AAAAAAADdwQAAAAAeHQACWJpcnRoZGF0ZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGVUeXBlni1SrhB9yq8CAAFMAAdmb3JtYXRzdAAQTGphdmEvdXRpbC9MaXN0O3hxAH4AFX5xAH4AC3QACVRJTUVTVEFNUH5xAH4AG3QABERhdGVxAH4AIHNxAH4AAAAAAAF3BAAAAAB4dAAGZ2VuZGVyc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABVxAH4AEH5xAH4AG3QAB0tleXdvcmRxAH4AIHh0AARjaXR5cQB+ABB0AAhsYXN0bmFtZXEAfgAQdAAHYmFsYW5jZXEAfgANdAAIZW1wbG95ZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABXN0YXRlc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnEAfgAxcQB+ADJ4dAADYWdlfnEAfgALdAAHSU5URUdFUnQABWVtYWlsc3EAfgATcQB+ABlxAH4AHHEAfgAgcQB+ACR0AARtYWxlfnEAfgALdAAHQk9PTEVBTngAeA==\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n"}} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.json index e4b32a61e8a..e69de29bb2d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.json @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(sum(balance)=[$1], sum(balance + 100)=[$2], sum(balance - 100)=[$3], sum(balance * 100)=[$4], sum(balance / 100)=[$5], gender=[$0])\n LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)=[SUM($2)], sum(balance - 100)=[SUM($3)], sum(balance * 100)=[SUM($4)], sum(balance / 100)=[SUM($5)])\n LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)=SUM($2),sum(balance - 100)=SUM($3),sum(balance * 100)=SUM($4),sum(balance / 100)=SUM($5)), PROJECT->[sum(balance), sum(balance + 100), sum(balance - 100), sum(balance * 100), sum(balance / 100), gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"sum(balance)\":{\"sum\":{\"field\":\"balance\"}},\"sum(balance + 100)\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0AQN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiKyIsCiAgICAia2luZCI6ICJQTFVTIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiA3LAogICAgICAibmFtZSI6ICIkNyIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogMTAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAYdwgAAAAgAAAADXQADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZX5xAH4AC3QABlNUUklOR3QAB2FkZHJlc3NzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AFHhwfnEAfgALdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAx0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAfeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAhAAAAAHNxAH4AAAAAAAN3BAAAAAB4dAAJYmlydGhkYXRlc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0ZVR5cGWeLVKuEH3KrwIAAUwAB2Zvcm1hdHN0ABBMamF2YS91dGlsL0xpc3Q7eHEAfgAVfnEAfgALdAAJVElNRVNUQU1QfnEAfgAbdAAERGF0ZXEAfgAgc3EAfgAAAAAAAXcEAAAAAHh0AAZnZW5kZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AFXEAfgAQfnEAfgAbdAAHS2V5d29yZHEAfgAgeHQABGNpdHlxAH4AEHQACGxhc3RuYW1lcQB+ABB0AAdiYWxhbmNlcQB+AA10AAhlbXBsb3llcnNxAH4AE3EAfgAZcQB+ABxxAH4AIHEAfgAkdAAFc3RhdGVzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACcQB+ADFxAH4AMnh0AANhZ2V+cQB+AAt0AAdJTlRFR0VSdAAFZW1haWxzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABG1hbGV+cQB+AAt0AAdCT09MRUFOeAB4\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}}}},\"sum(balance - 100)\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0AUB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiLSIsCiAgICAia2luZCI6ICJNSU5VUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogNywKICAgICAgIm5hbWUiOiAiJDciCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDEwMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0sCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0KfXQACmZpZWxkVHlwZXNzcgAXamF2YS51dGlsLkxpbmtlZEhhc2hNYXA0wE5cEGzA+wIAAVoAC2FjY2Vzc09yZGVyeHIAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAAGHcIAAAAIAAAAA10AA5hY2NvdW50X251bWJlcn5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd0AAlmaXJzdG5hbWV+cQB+AAt0AAZTVFJJTkd0AAdhZGRyZXNzc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+ABR4cH5xAH4AC3QAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgAMdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AH3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AIQAAAABzcQB+AAAAAAADdwQAAAAAeHQACWJpcnRoZGF0ZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGVUeXBlni1SrhB9yq8CAAFMAAdmb3JtYXRzdAAQTGphdmEvdXRpbC9MaXN0O3hxAH4AFX5xAH4AC3QACVRJTUVTVEFNUH5xAH4AG3QABERhdGVxAH4AIHNxAH4AAAAAAAF3BAAAAAB4dAAGZ2VuZGVyc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABVxAH4AEH5xAH4AG3QAB0tleXdvcmRxAH4AIHh0AARjaXR5cQB+ABB0AAhsYXN0bmFtZXEAfgAQdAAHYmFsYW5jZXEAfgANdAAIZW1wbG95ZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABXN0YXRlc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnEAfgAxcQB+ADJ4dAADYWdlfnEAfgALdAAHSU5URUdFUnQABWVtYWlsc3EAfgATcQB+ABlxAH4AHHEAfgAgcQB+ACR0AARtYWxlfnEAfgALdAAHQk9PTEVBTngAeA==\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}}}},\"sum(balance * 100)\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0AQR7CiAgIm9wIjogewogICAgIm5hbWUiOiAiKiIsCiAgICAia2luZCI6ICJUSU1FUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogNywKICAgICAgIm5hbWUiOiAiJDciCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDEwMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgAXamF2YS51dGlsLkxpbmtlZEhhc2hNYXA0wE5cEGzA+wIAAVoAC2FjY2Vzc09yZGVyeHIAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAAGHcIAAAAIAAAAA10AA5hY2NvdW50X251bWJlcn5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd0AAlmaXJzdG5hbWV+cQB+AAt0AAZTVFJJTkd0AAdhZGRyZXNzc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+ABR4cH5xAH4AC3QAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgAMdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AH3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AIQAAAABzcQB+AAAAAAADdwQAAAAAeHQACWJpcnRoZGF0ZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGVUeXBlni1SrhB9yq8CAAFMAAdmb3JtYXRzdAAQTGphdmEvdXRpbC9MaXN0O3hxAH4AFX5xAH4AC3QACVRJTUVTVEFNUH5xAH4AG3QABERhdGVxAH4AIHNxAH4AAAAAAAF3BAAAAAB4dAAGZ2VuZGVyc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABVxAH4AEH5xAH4AG3QAB0tleXdvcmRxAH4AIHh0AARjaXR5cQB+ABB0AAhsYXN0bmFtZXEAfgAQdAAHYmFsYW5jZXEAfgANdAAIZW1wbG95ZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABXN0YXRlc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnEAfgAxcQB+ADJ4dAADYWdlfnEAfgALdAAHSU5URUdFUnQABWVtYWlsc3EAfgATcQB+ABlxAH4AHHEAfgAgcQB+ACR0AARtYWxlfnEAfgALdAAHQk9PTEVBTngAeA==\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}}}},\"sum(balance / 100)\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0Ac97CiAgIm9wIjogewogICAgIm5hbWUiOiAiRElWSURFIiwKICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDcsCiAgICAgICJuYW1lIjogIiQ3IgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAxMDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdLAogICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAibnVsbGFibGUiOiB0cnVlCiAgfSwKICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgImR5bmFtaWMiOiBmYWxzZQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAYdwgAAAAgAAAADXQADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZX5xAH4AC3QABlNUUklOR3QAB2FkZHJlc3NzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AFHhwfnEAfgALdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAx0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAfeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAhAAAAAHNxAH4AAAAAAAN3BAAAAAB4dAAJYmlydGhkYXRlc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0ZVR5cGWeLVKuEH3KrwIAAUwAB2Zvcm1hdHN0ABBMamF2YS91dGlsL0xpc3Q7eHEAfgAVfnEAfgALdAAJVElNRVNUQU1QfnEAfgAbdAAERGF0ZXEAfgAgc3EAfgAAAAAAAXcEAAAAAHh0AAZnZW5kZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AFXEAfgAQfnEAfgAbdAAHS2V5d29yZHEAfgAgeHQABGNpdHlxAH4AEHQACGxhc3RuYW1lcQB+ABB0AAdiYWxhbmNlcQB+AA10AAhlbXBsb3llcnNxAH4AE3EAfgAZcQB+ABxxAH4AIHEAfgAkdAAFc3RhdGVzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACcQB+ADFxAH4AMnh0AANhZ2V+cQB+AAt0AAdJTlRFR0VSdAAFZW1haWxzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABG1hbGV+cQB+AAt0AAdCT09MRUFOeAB4\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml new file mode 100644 index 00000000000..dd8a0fac298 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(sum(balance)=[$1], sum(balance + 100)=[$2], sum(balance - 100)=[$3], sum(balance * 100)=[$4], sum(balance / 100)=[$5], gender=[$0]) + LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)=[SUM($2)], sum(balance - 100)=[SUM($3)], sum(balance * 100)=[SUM($4)], sum(balance / 100)=[SUM($5)]) + LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0Ac97CiAgIm9wIjogewogICAgIm5hbWUiOiAiRElWSURFIiwKICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDcsCiAgICAgICJuYW1lIjogIiQ3IgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAxMDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdLAogICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAibnVsbGFibGUiOiB0cnVlCiAgfSwKICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgImR5bmFtaWMiOiBmYWxzZQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAYdwgAAAAgAAAADXQADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZX5xAH4AC3QABlNUUklOR3QAB2FkZHJlc3NzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AFHhwfnEAfgALdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAx0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAfeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAhAAAAAHNxAH4AAAAAAAN3BAAAAAB4dAAJYmlydGhkYXRlc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0ZVR5cGWeLVKuEH3KrwIAAUwAB2Zvcm1hdHN0ABBMamF2YS91dGlsL0xpc3Q7eHEAfgAVfnEAfgALdAAJVElNRVNUQU1QfnEAfgAbdAAERGF0ZXEAfgAgc3EAfgAAAAAAAXcEAAAAAHh0AAZnZW5kZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AFXEAfgAQfnEAfgAbdAAHS2V5d29yZHEAfgAgeHQABGNpdHlxAH4AEHQACGxhc3RuYW1lcQB+ABB0AAdiYWxhbmNlcQB+AA10AAhlbXBsb3llcnNxAH4AE3EAfgAZcQB+ABxxAH4AIHEAfgAkdAAFc3RhdGVzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACcQB+ADFxAH4AMnh0AANhZ2V+cQB+AAt0AAdJTlRFR0VSdAAFZW1haWxzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABG1hbGV+cQB+AAt0AAdCT09MRUFOeAB4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_aligntime.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_aligntime.json deleted file mode 100644 index e66d4edbffb..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_aligntime.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(category=[$0], value=[$1], timestamp=[$2], @timestamp=[$9])\n LogicalSort(fetch=[5])\n LogicalProject(category=[$1], value=[$2], timestamp=[$3], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9], @timestamp=[FROM_UNIXTIME(*(*(FLOOR(/(/(UNIX_TIMESTAMP($0), 3600), 2)), 2), 3600))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]])\n","physical":"EnumerableCalc(expr#0..3=[{inputs}], expr#4=[UNIX_TIMESTAMP($t3)], expr#5=[3600], expr#6=[/($t4, $t5)], expr#7=[2], expr#8=[/($t6, $t7)], expr#9=[FLOOR($t8)], expr#10=[*($t9, $t7)], expr#11=[*($t10, $t5)], expr#12=[FROM_UNIXTIME($t11)], proj#0..2=[{exprs}], $f3=[$t12])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[LIMIT->5, LIMIT->10000, PROJECT->[category, value, timestamp, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"category\",\"value\",\"timestamp\",\"@timestamp\"],\"excludes\":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)])\n"}} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_aligntime.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_aligntime.yaml new file mode 100644 index 00000000000..53cd91e02ca --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_aligntime.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(category=[$0], value=[$1], timestamp=[$2], @timestamp=[$9]) + LogicalSort(fetch=[5]) + LogicalProject(category=[$1], value=[$2], timestamp=[$3], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9], @timestamp=[FROM_UNIXTIME(*(*(FLOOR(/(/(UNIX_TIMESTAMP($0), 3600), 2)), 2), 3600))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + physical: | + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[UNIX_TIMESTAMP($t3)], expr#5=[3600], expr#6=[/($t4, $t5)], expr#7=[2], expr#8=[/($t6, $t7)], expr#9=[FLOOR($t8)], expr#10=[*($t9, $t7)], expr#11=[*($t10, $t5)], expr#12=[FROM_UNIXTIME($t11)], proj#0..2=[{exprs}], $f3=[$t12]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[PROJECT->[category, value, timestamp, @timestamp], LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["category","value","timestamp","@timestamp"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_span.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_span.json deleted file mode 100644 index 13daa0ae3cb..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_span.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[SPAN_BUCKET($8, 10)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"EnumerableCalc(expr#0..10=[{inputs}], expr#11=[10], expr#12=[SPAN_BUCKET($t10, $t11)], proj#0..9=[{exprs}], $f10=[$t12])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[LIMIT->5, LIMIT->10000, PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, email, lastname, age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"email\",\"lastname\",\"age\"],\"excludes\":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)])\n"}} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_span.yaml new file mode 100644 index 00000000000..3a1cee955d3 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_bin_span.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[SPAN_BUCKET($8, 10)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..10=[{inputs}], expr#11=[10], expr#12=[SPAN_BUCKET($t10, $t11)], proj#0..9=[{exprs}], $f10=[$t12]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, email, lastname, age], LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","email","lastname","age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.json deleted file mode 100644 index aa12f728621..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age=[$8])\n LogicalFilter(condition=[>($8, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($8, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[state, city, avg_age], PROJECT->[avg_age, state, city], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.yaml new file mode 100644 index 00000000000..6982cd13bf5 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg_age=[$2], state=[$0], city=[$1]) + LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) + LogicalProject(state=[$7], city=[$5], age=[$8]) + LogicalFilter(condition=[>($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[city, state, age], FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state, city], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"_source":{"includes":["city","state","age"],"excludes":[]},"sort":[],"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_cost_w_pushdown.txt b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_cost_w_pushdown.txt index 7848bd12b4f..db6e7a8654d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_cost_w_pushdown.txt +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_cost_w_pushdown.txt @@ -1 +1 @@ -CalciteEnumerableIndexScan(table=[[OpenSearch, test]], PushDownContext=[[PROJECT->[name, age], FILTER->=($1, 20), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"term\":{\"age\":{\"value\":20,\"boost\":1.0}}},\"_source\":{\"includes\":[\"name\",\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]): rowcount = 1093.5, cumulative cost = {1093.5 rows, 1094.5 cpu, 0.0 io} +CalciteEnumerableIndexScan(table=[[OpenSearch, test]], PushDownContext=[[PROJECT->[name, age], FILTER->=($1, 20), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"term\":{\"age\":{\"value\":20,\"boost\":1.0}}},\"_source\":{\"includes\":[\"name\",\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]): rowcount = 1500.0, cumulative cost = {2698.2000000000003 rows, 0.0 cpu, 0.0 io} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_cost_wo_pushdown.txt b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_cost_wo_pushdown.txt index 36c10d64b0c..fbd02b49746 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_cost_wo_pushdown.txt +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_cost_wo_pushdown.txt @@ -1 +1 @@ -CalciteEnumerableIndexScan(table=[[OpenSearch, test]]): rowcount = 10000.0, cumulative cost = {10000.0 rows, 10001.0 cpu, 0.0 io} \ No newline at end of file +CalciteEnumerableIndexScan(table=[[OpenSearch, test]]): rowcount = 10000.0, cumulative cost = {72000.0 rows, 0.0 cpu, 0.0 io} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push.json deleted file mode 100644 index 602c4aba839..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalFilter(condition=[>($3, 10000)])\n LogicalFilter(condition=[<($8, 40)])\n LogicalFilter(condition=[>($8, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[balance, age], FILTER->>($1, 30), FILTER-><($1, 40), FILTER->>($0, 10000), PROJECT->[age], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":null,\"to\":40,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},{\"range\":{\"balance\":{\"from\":10000,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push.yaml new file mode 100644 index 00000000000..f09212ae6a1 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalFilter(condition=[>($3, 10000)]) + LogicalFilter(condition=[<($8, 40)]) + LogicalFilter(condition=[>($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[balance, age], FILTER->AND(SEARCH($1, Sarg[(30..40)]), >($0, 10000)), PROJECT->[age], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"range":{"age":{"from":30.0,"to":40.0,"include_lower":false,"include_upper":false,"boost":1.0}}},{"range":{"balance":{"from":10000,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["age"],"excludes":[]},"sort":[{"_doc":{"order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_date_string.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_date_string.json deleted file mode 100644 index 14c835be324..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_date_string.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalFilter(condition=[<($0, DATE('2018-11-09 00:00:00.000000000':VARCHAR))])\n LogicalFilter(condition=[>($0, DATE('2016-12-08 00:00:00.123456789':VARCHAR))])\n LogicalProject(yyyy-MM-dd=[$83])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]], PushDownContext=[[PROJECT->[yyyy-MM-dd], FILTER->>($0, '2016-12-08'), FILTER-><($0, '2018-11-09'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\":{\"yyyy-MM-dd\":{\"from\":\"2016-12-08\",\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"yyyy-MM-dd\":{\"from\":null,\"to\":\"2018-11-09\",\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"yyyy-MM-dd\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_date_string.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_date_string.yaml new file mode 100644 index 00000000000..10785345f09 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_date_string.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalFilter(condition=[<($0, DATE('2018-11-09 00:00:00.000000000':VARCHAR))]) + LogicalFilter(condition=[>($0, DATE('2016-12-08 00:00:00.123456789':VARCHAR))]) + LogicalProject(yyyy-MM-dd=[$83]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]], PushDownContext=[[PROJECT->[yyyy-MM-dd], FILTER->SEARCH($0, Sarg[('2016-12-08':VARCHAR..'2018-11-09':VARCHAR)]:VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"yyyy-MM-dd":{"from":"2016-12-08","to":"2018-11-09","include_lower":false,"include_upper":false,"boost":1.0}}},"_source":{"includes":["yyyy-MM-dd"],"excludes":[]},"sort":[{"_doc":{"order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_time_string.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_time_string.json deleted file mode 100644 index 7c4e8dce916..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_time_string.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalFilter(condition=[<($0, TIME('2018-11-09 19:00:00.123456789':VARCHAR))])\n LogicalFilter(condition=[>($0, TIME('2016-12-08 12:00:00.123456789':VARCHAR))])\n LogicalProject(custom_time=[$49])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]], PushDownContext=[[PROJECT->[custom_time], FILTER->>($0, '12:00:00.123456789'), FILTER-><($0, '19:00:00.123456789'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\":{\"custom_time\":{\"from\":\"12:00:00.123456789\",\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"custom_time\":{\"from\":null,\"to\":\"19:00:00.123456789\",\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"custom_time\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_time_string.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_time_string.yaml new file mode 100644 index 00000000000..6ba1a3a75bb --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_time_string.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalFilter(condition=[<($0, TIME('2018-11-09 19:00:00.123456789':VARCHAR))]) + LogicalFilter(condition=[>($0, TIME('2016-12-08 12:00:00.123456789':VARCHAR))]) + LogicalProject(custom_time=[$49]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]], PushDownContext=[[PROJECT->[custom_time], FILTER->SEARCH($0, Sarg[('12:00:00.123456789':VARCHAR..'19:00:00.123456789':VARCHAR)]:VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"custom_time":{"from":"12:00:00.123456789","to":"19:00:00.123456789","include_lower":false,"include_upper":false,"boost":1.0}}},"_source":{"includes":["custom_time"],"excludes":[]},"sort":[{"_doc":{"order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_timestamp_string.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_timestamp_string.json deleted file mode 100644 index e63992dbf56..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_timestamp_string.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12])\n LogicalFilter(condition=[<($3, TIMESTAMP('2018-11-09 00:00:00.000000000':VARCHAR))])\n LogicalFilter(condition=[>($3, TIMESTAMP('2016-12-08 00:00:00.000000000':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], FILTER->>($3, '2016-12-08 00:00:00'), FILTER-><($3, '2018-11-09 00:00:00'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\":{\"birthdate\":{\"from\":\"2016-12-08T00:00:00.000Z\",\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"birthdate\":{\"from\":null,\"to\":\"2018-11-09T00:00:00.000Z\",\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"birthdate\",\"gender\",\"city\",\"lastname\",\"balance\",\"employer\",\"state\",\"age\",\"email\",\"male\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_timestamp_string.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_timestamp_string.yaml new file mode 100644 index 00000000000..045b566a0c8 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_push_compare_timestamp_string.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + LogicalFilter(condition=[<($3, TIMESTAMP('2018-11-09 00:00:00.000000000':VARCHAR))]) + LogicalFilter(condition=[>($3, TIMESTAMP('2016-12-08 00:00:00.000000000':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], FILTER->SEARCH($3, Sarg[('2016-12-08 00:00:00':VARCHAR..'2018-11-09 00:00:00':VARCHAR)]:VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"range":{"birthdate":{"from":"2016-12-08T00:00:00.000Z","to":"2018-11-09T00:00:00.000Z","include_lower":false,"include_upper":false,"boost":1.0}}},"_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]},"sort":[{"_doc":{"order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_then_limit_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_then_limit_push.json deleted file mode 100644 index ff014daca5f..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_then_limit_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalSort(fetch=[5])\n LogicalFilter(condition=[>($8, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($8, 30), LIMIT->5, PROJECT->[age], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_then_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_then_limit_push.yaml new file mode 100644 index 00000000000..46303e697c8 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_then_limit_push.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalSort(fetch=[5]) + LogicalFilter(condition=[>($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], FILTER->>($0, 30), LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"_source":{"includes":["age"],"excludes":[]},"sort":[{"_doc":{"order":"asc"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10_5_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10_5_push.json deleted file mode 100644 index 44289c5dd6a..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10_5_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalSort(fetch=[5])\n LogicalSort(fetch=[10])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[LIMIT->10, LIMIT->5, PROJECT->[age], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10_5_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10_5_push.yaml new file mode 100644 index 00000000000..dee2d92c7a3 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10_5_push.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalSort(fetch=[5]) + LogicalSort(fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->10, LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10_filter_5_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10_filter_5_push.json deleted file mode 100644 index 8dc3879c16d..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10_filter_5_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalSort(fetch=[5])\n LogicalFilter(condition=[>($8, 30)])\n LogicalSort(fetch=[10])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableLimit(fetch=[5])\n EnumerableCalc(expr#0=[{inputs}], expr#1=[30], expr#2=[>($t0, $t1)], age=[$t0], $condition=[$t2])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[LIMIT->10, PROJECT->[age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10_filter_5_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10_filter_5_push.yaml new file mode 100644 index 00000000000..21f46d7f96a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10_filter_5_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalSort(fetch=[5]) + LogicalFilter(condition=[>($8, 30)]) + LogicalSort(fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(fetch=[5]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[30], expr#2=[>($t0, $t1)], age=[$t0], $condition=[$t2]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->10], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10from1_10from2_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10from1_10from2_push.json deleted file mode 100644 index 80fa609f438..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10from1_10from2_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalSort(offset=[2], fetch=[10])\n LogicalSort(offset=[1], fetch=[10])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[LIMIT->[10 from 1], PROJECT->[age], LIMIT->[10 from 2], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":3,\"size\":8,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, requestedTotalSize=8, pageSize=null, startFrom=3)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10from1_10from2_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10from1_10from2_push.yaml new file mode 100644 index 00000000000..e17b61abc78 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_10from1_10from2_push.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalSort(offset=[2], fetch=[10]) + LogicalSort(offset=[1], fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->[10 from 1], LIMIT->[10 from 2], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":3,"size":8,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=8, pageSize=null, startFrom=3)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_5_10_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_5_10_push.json deleted file mode 100644 index c7e9a7e6d82..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_5_10_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalSort(fetch=[10])\n LogicalSort(fetch=[5])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[LIMIT->5, PROJECT->[age], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_5_10_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_5_10_push.yaml new file mode 100644 index 00000000000..7b936024c74 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_5_10_push.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalSort(fetch=[10]) + LogicalSort(fetch=[5]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5, LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown2.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown2.json deleted file mode 100644 index 391568b1234..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown2.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(fetch=[100])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->100, PROJECT->[count(), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":100,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown2.yaml new file mode 100644 index 00000000000..3a19f39e7da --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown2.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(fetch=[100]) + LogicalProject(count()=[$1], state=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(state=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state], LIMIT->100, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":100,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_offsets_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_offsets_push.json deleted file mode 100644 index a0138cedd1f..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_offsets_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalSort(offset=[2], fetch=[5])\n LogicalSort(offset=[1], fetch=[10])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[LIMIT->[10 from 1], LIMIT->[5 from 2], PROJECT->[age], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":3,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, requestedTotalSize=5, pageSize=null, startFrom=3)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_offsets_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_offsets_push.yaml new file mode 100644 index 00000000000..1a5003d8a2b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_offsets_push.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalSort(offset=[2], fetch=[5]) + LogicalSort(offset=[1], fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->[10 from 1], LIMIT->[5 from 2], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":3,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=3)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.json deleted file mode 100644 index 4faa15d81ff..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(ageMinus=[$17])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], ageMinus=[-($8, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableCalc(expr#0=[{inputs}], expr#1=[30], expr#2=[-($t0, $t1)], $f0=[$t2])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[LIMIT->5, LIMIT->10000, PROJECT->[age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.yaml new file mode 100644 index 00000000000..187f948a6e4 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_push.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(ageMinus=[$17]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], ageMinus=[-($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0=[{inputs}], expr#1=[30], expr#2=[-($t0, $t1)], $f0=[$t2]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_filter_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_filter_push.json deleted file mode 100644 index 48b7cea9fd0..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_filter_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalFilter(condition=[>($8, 30)])\n LogicalSort(fetch=[5])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0=[{inputs}], expr#1=[30], expr#2=[>($t0, $t1)], age=[$t0], $condition=[$t2])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[LIMIT->5, PROJECT->[age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_filter_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_filter_push.yaml new file mode 100644 index 00000000000..aed8746d78b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_filter_push.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalFilter(condition=[>($8, 30)]) + LogicalSort(fetch=[5]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[30], expr#2=[>($t0, $t1)], age=[$t0], $condition=[$t2]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_sort_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_sort_push.json deleted file mode 100644 index f73adcb19a9..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_sort_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalSort(sort0=[$8], dir0=[ASC-nulls-first])\n LogicalSort(fetch=[5])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[LIMIT->5, PROJECT->[age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_sort_push.yaml new file mode 100644 index 00000000000..1fdacd0c05b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_then_sort_push.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalSort(sort0=[$8], dir0=[ASC-nulls-first]) + LogicalSort(fetch=[5]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_multi_agg_tobe_combined.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_multi_agg_tobe_combined.yaml new file mode 100644 index 00000000000..61724171f7a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_multi_agg_tobe_combined.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(c1=[$1], c2=[$1], gender=[$0]) + LogicalAggregate(group=[{0}], c1=[COUNT($1)]) + LogicalProject(gender=[$4], balance=[$3]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..1=[{inputs}], c1=[$t1], c2=[$t1], gender=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c1=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"c1":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_output.json deleted file mode 100644 index 973efe47fa9..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_output.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age2=[$2])\n LogicalFilter(condition=[<=($3, 1)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2 ORDER BY $2)])\n LogicalFilter(condition=[IS NOT NULL($2)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age=[$8])\n LogicalFilter(condition=[>($8, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableCalc(expr#0..1=[{inputs}], age2=[$t0])\n EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[<=($t1, $t2)], proj#0..1=[{exprs}], $condition=[$t3])\n EnumerableWindow(window#0=[window(partition {0} order by [0] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])])\n EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], expr#3=[IS NOT NULL($t0)], age2=[$t2], $condition=[$t3])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($8, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[state, city, avg_age], SORT->[0 ASC FIRST], PROJECT->[avg_age, state], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_output.yaml new file mode 100644 index 00000000000..bbeefb56be3 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output.yaml @@ -0,0 +1,21 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age2=[$2]) + LogicalFilter(condition=[<=($3, 1)]) + LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2 ORDER BY $2)]) + LogicalFilter(condition=[IS NOT NULL($2)]) + LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) + LogicalProject(avg_age=[$2], state=[$0], city=[$1]) + LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) + LogicalProject(state=[$7], city=[$5], age=[$8]) + LogicalFilter(condition=[>($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..1=[{inputs}], age2=[$t0]) + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[<=($t1, $t2)], proj#0..1=[{exprs}], $condition=[$t3]) + EnumerableWindow(window#0=[window(partition {0} order by [0] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], expr#3=[IS NOT NULL($t0)], age2=[$t2], $condition=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[city, state, age], FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), SORT->[0 ASC FIRST], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"_source":{"includes":["city","state","age"],"excludes":[]},"sort":[],"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.json deleted file mode 100644 index 2716bfe9e83..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(patterns_field=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'pattern'))], pattern_count=[$1], tokens=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'tokens'))], sample_logs=[$2])\n LogicalAggregate(group=[{1}], pattern_count=[COUNT($1)], sample_logs=[TAKE($0, $2)])\n LogicalProject(email=[$9], patterns_field=[REGEXP_REPLACE($9, '[a-zA-Z0-9]+':VARCHAR, '<*>')], $f18=[10])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableCalc(expr#0..2=[{inputs}], expr#3=[PATTERN_PARSER($t0, $t1)], expr#4=['pattern'], expr#5=[ITEM($t3, $t4)], expr#6=[SAFE_CAST($t5)], expr#7=['tokens'], expr#8=[ITEM($t3, $t7)], expr#9=[SAFE_CAST($t8)], $f0=[$t6], pattern_count=[$t2], $f2=[$t9], sample_logs=[$t1])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},pattern_count=COUNT($1),sample_logs=TAKE($0, $2)), LIMIT->10000, PROJECT->[patterns_field, sample_logs, pattern_count]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"patterns_field\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBx3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiA5LAogICAgICAibmFtZSI6ICIkOSIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIlthLXpBLVowLTldKyIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICI8Kj4iLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogMwogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"pattern_count\":{\"value_count\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBx3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiA5LAogICAgICAibmFtZSI6ICIkOSIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIlthLXpBLVowLTldKyIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICI8Kj4iLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogMwogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}}}},\"sample_logs\":{\"top_hits\":{\"from\":0,\"size\":10,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"email\"],\"excludes\":[]}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml new file mode 100644 index 00000000000..59f80796fa7 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(patterns_field=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'pattern'))], pattern_count=[$1], tokens=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'tokens'))], sample_logs=[$2]) + LogicalAggregate(group=[{1}], pattern_count=[COUNT($1)], sample_logs=[TAKE($0, $2)]) + LogicalProject(email=[$9], patterns_field=[REGEXP_REPLACE($9, '[a-zA-Z0-9]+':VARCHAR, '<*>')], $f18=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[PATTERN_PARSER($t0, $t2)], expr#4=['pattern'], expr#5=[ITEM($t3, $t4)], expr#6=[SAFE_CAST($t5)], expr#7=['tokens'], expr#8=[ITEM($t3, $t7)], expr#9=[SAFE_CAST($t8)], patterns_field=[$t6], pattern_count=[$t1], tokens=[$t9], sample_logs=[$t2]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},pattern_count=COUNT($1),sample_logs=TAKE($0, $2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"patterns_field":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBx3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiA5LAogICAgICAibmFtZSI6ICIkOSIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIlthLXpBLVowLTldKyIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICI8Kj4iLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogMwogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"pattern_count":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBx3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiA5LAogICAgICAibmFtZSI6ICIkOSIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIlthLXpBLVowLTldKyIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICI8Kj4iLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogMwogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}},"sample_logs":{"top_hits":{"from":0,"size":10,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["email"],"excludes":[]}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_regex.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_regex.json deleted file mode 100644 index a1a971f290e..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_regex.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10])\n LogicalSort(fetch=[5])\n LogicalFilter(condition=[REGEXP_CONTAINS($10, '^[A-Z][a-z]+$':VARCHAR)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[SCRIPT->REGEXP_CONTAINS($10, '^[A-Z][a-z]+$':VARCHAR), LIMIT->5, PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBRHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfQ09OVEFJTlMiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMTAsCiAgICAgICJuYW1lIjogIiQxMCIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIl5bQS1aXVthLXpdKyQiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAQeHB+cQB+AAp0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4AC3QABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABt4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB0AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABF+cQB+AAp0AAZTVFJJTkd+cQB+ABd0AAdLZXl3b3JkcQB+ABx4dAAHYWRkcmVzc3NxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgAMdAAGZ2VuZGVyc3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAnEAfgAhcQB+ACJ4dAAEY2l0eXNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHQACGVtcGxveWVyc3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAnEAfgAhcQB+ACJ4dAAFc3RhdGVzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAACcQB+ACFxAH4AInh0AANhZ2VxAH4ADHQABWVtYWlsc3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAnEAfgAhcQB+ACJ4dAAIbGFzdG5hbWVzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAACcQB+ACFxAH4AInh4eA==\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)])\n"}} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_regex.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_regex.yaml new file mode 100644 index 00000000000..076f946aca3 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_regex.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalSort(fetch=[5]) + LogicalFilter(condition=[REGEXP_CONTAINS($10, '^[A-Z][a-z]+$':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->REGEXP_CONTAINS($10, '^[A-Z][a-z]+$':VARCHAR), LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQEj3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AUR7CiAgIm9wIjogewogICAgIm5hbWUiOiAiUkVHRVhQX0NPTlRBSU5TIiwKICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDEwLAogICAgICAibmFtZSI6ICIkMTAiCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICJeW0EtWl1bYS16XSskIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAt0AA5hY2NvdW50X251bWJlcn5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd0AAlmaXJzdG5hbWVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AEHhwfnEAfgAKdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAt0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAbeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAdAAAAAHNxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgARfnEAfgAKdAAGU1RSSU5HfnEAfgAXdAAHS2V5d29yZHEAfgAceHQAB2FkZHJlc3NzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAAAeHQAB2JhbGFuY2VxAH4ADHQABmdlbmRlcnNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHQABGNpdHlzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAACcQB+ACFxAH4AInh0AAhlbXBsb3llcnNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHQABXN0YXRlc3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAnEAfgAhcQB+ACJ4dAADYWdlcQB+AAx0AAVlbWFpbHNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHQACGxhc3RuYW1lc3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAnEAfgAhcQB+ACJ4eHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]},"sort":[{"_doc":{"order":"asc"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_regex_negated.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_regex_negated.json deleted file mode 100644 index 254093b542a..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_regex_negated.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10])\n LogicalSort(fetch=[5])\n LogicalFilter(condition=[NOT(REGEXP_CONTAINS($10, '.*son$':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[SCRIPT->NOT(REGEXP_CONTAINS($10, '.*son$':VARCHAR)), LIMIT->5, PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQB9XsKICAib3AiOiB7CiAgICAibmFtZSI6ICJOT1QiLAogICAgImtpbmQiOiAiTk9UIiwKICAgICJzeW50YXgiOiAiUFJFRklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiUkVHRVhQX0NPTlRBSU5TIiwKICAgICAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDEwLAogICAgICAgICAgIm5hbWUiOiAiJDEwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiLipzb24kIiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAQeHB+cQB+AAp0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4AC3QABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABt4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB0AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABF+cQB+AAp0AAZTVFJJTkd+cQB+ABd0AAdLZXl3b3JkcQB+ABx4dAAHYWRkcmVzc3NxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgAMdAAGZ2VuZGVyc3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAnEAfgAhcQB+ACJ4dAAEY2l0eXNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHQACGVtcGxveWVyc3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAnEAfgAhcQB+ACJ4dAAFc3RhdGVzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAACcQB+ACFxAH4AInh0AANhZ2VxAH4ADHQABWVtYWlsc3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAnEAfgAhcQB+ACJ4dAAIbGFzdG5hbWVzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAACcQB+ACFxAH4AInh4eA==\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)])\n"}} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_regex_negated.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_regex_negated.yaml new file mode 100644 index 00000000000..ab766896449 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_regex_negated.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalSort(fetch=[5]) + LogicalFilter(condition=[NOT(REGEXP_CONTAINS($10, '.*son$':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->NOT(REGEXP_CONTAINS($10, '.*son$':VARCHAR)), LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQEj3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AfV7CiAgIm9wIjogewogICAgIm5hbWUiOiAiTk9UIiwKICAgICJraW5kIjogIk5PVCIsCiAgICAic3ludGF4IjogIlBSRUZJWCIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlJFR0VYUF9DT05UQUlOUyIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiAxMCwKICAgICAgICAgICJuYW1lIjogIiQxMCIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogIi4qc29uJCIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAt0AA5hY2NvdW50X251bWJlcn5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd0AAlmaXJzdG5hbWVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AEHhwfnEAfgAKdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAt0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAbeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAdAAAAAHNxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgARfnEAfgAKdAAGU1RSSU5HfnEAfgAXdAAHS2V5d29yZHEAfgAceHQAB2FkZHJlc3NzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAAAeHQAB2JhbGFuY2VxAH4ADHQABmdlbmRlcnNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHQABGNpdHlzcQB+AA9xAH4AFXEAfgAYcQB+ABxzcQB+AAAAAAADdwQAAAACcQB+ACFxAH4AInh0AAhlbXBsb3llcnNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHQABXN0YXRlc3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAnEAfgAhcQB+ACJ4dAADYWdlcQB+AAx0AAVlbWFpbHNxAH4AD3EAfgAVcQB+ABhxAH4AHHNxAH4AAAAAAAN3BAAAAAJxAH4AIXEAfgAieHQACGxhc3RuYW1lc3EAfgAPcQB+ABVxAH4AGHEAfgAcc3EAfgAAAAAAA3cEAAAAAnEAfgAhcQB+ACJ4eHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]},"sort":[{"_doc":{"order":"asc"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_rex.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_rex.json deleted file mode 100644 index 250524b8457..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_rex.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], initial=[$17])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], initial=[REX_EXTRACT($10, '(?^[A-Z])', 1)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableCalc(expr#0..10=[{inputs}], expr#11=['(?^[A-Z])'], expr#12=[1], expr#13=[REX_EXTRACT($t10, $t11, $t12)], proj#0..10=[{exprs}], $f11=[$t13])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[LIMIT->5, LIMIT->10000, PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_rex.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_rex.yaml new file mode 100644 index 00000000000..3658fe35f41 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_rex.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], initial=[$17]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], initial=[REX_EXTRACT($10, '(?^[A-Z])', 1)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..10=[{inputs}], expr#11=['(?^[A-Z])'], expr#12=[1], expr#13=[REX_EXTRACT($t10, $t11, $t12)], proj#0..10=[{exprs}], $f11=[$t13]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_count_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_count_push.json deleted file mode 100644 index afc38905df1..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_count_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalSort(sort0=[$8], dir0=[ASC-nulls-first], fetch=[5])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[SORT->[{\n \"age\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], LIMIT->5, PROJECT->[age], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_count_push.yaml new file mode 100644 index 00000000000..ea09a6f976f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_count_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalSort(sort0=[$8], dir0=[ASC-nulls-first], fetch=[5]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], SORT->[{ + "age" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]},"sort":[{"age":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_limit_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_limit_push.json deleted file mode 100644 index 2807f689d84..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_limit_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalSort(sort0=[$8], dir0=[ASC-nulls-first], fetch=[5])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[SORT->[{\n \"age\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], LIMIT->5, PROJECT->[age], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_limit_push.yaml new file mode 100644 index 00000000000..ea09a6f976f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_limit_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalSort(sort0=[$8], dir0=[ASC-nulls-first], fetch=[5]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], SORT->[{ + "age" : { + "order" : "asc", + "missing" : "_first" + } + }], LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]},"sort":[{"age":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.json deleted file mode 100644 index 8430c670a13..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC])\n LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[SUM($2)])\n LogicalProject(@timestamp=[$0], host=[CASE(IS NOT NULL($3), $1, CASE(IS NULL($1), null:NULL, 'OTHER'))], avg(cpu_usage)=[$2])\n LogicalJoin(condition=[=($1, $3)], joinType=[left])\n LogicalProject(@timestamp=[$1], host=[$0], $f2=[$2])\n LogicalAggregate(group=[{0, 2}], agg#0=[AVG($1)])\n LogicalProject(host=[$4], cpu_usage=[$7], $f3=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n LogicalSort(sort0=[$1], dir0=[DESC], fetch=[10])\n LogicalAggregate(group=[{1}], grand_total=[SUM($2)])\n LogicalFilter(condition=[IS NOT NULL($1)])\n LogicalProject(@timestamp=[$1], host=[$0], $f2=[$2])\n LogicalAggregate(group=[{0, 2}], agg#0=[AVG($1)])\n LogicalProject(host=[$4], cpu_usage=[$7], $f3=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC])\n EnumerableAggregate(group=[{0, 1}], avg(cpu_usage)=[SUM($2)])\n EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NOT NULL($t3)], expr#6=[IS NULL($t1)], expr#7=[null:NULL], expr#8=['OTHER'], expr#9=[CASE($t6, $t7, $t8)], expr#10=[CASE($t5, $t1, $t9)], @timestamp=[$t0], host=[$t10], avg(cpu_usage)=[$t2])\n EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left])\n EnumerableSort(sort0=[$1], dir0=[ASC])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], @timestamp=[$t1], host=[$t0], $f2=[$t8])\n EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, cpu_usage, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"host\",\"cpu_usage\",\"@timestamp\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableLimit(fetch=[10])\n EnumerableSort(sort0=[$1], dir0=[DESC])\n EnumerableAggregate(group=[{0}], grand_total=[SUM($2)])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], $f2=[$t8])\n EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host, cpu_usage], FILTER->IS NOT NULL($1), PROJECT->[host, cpu_usage, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"host\",\"boost\":1.0}},\"_source\":{\"includes\":[\"host\",\"cpu_usage\",\"@timestamp\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml new file mode 100644 index 00000000000..d901fbf22ac --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.yaml @@ -0,0 +1,37 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[SUM($2)]) + LogicalProject(@timestamp=[$0], host=[CASE(IS NOT NULL($3), $1, CASE(IS NULL($1), null:NULL, 'OTHER'))], avg(cpu_usage)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalProject(@timestamp=[$1], host=[$0], $f2=[$2]) + LogicalAggregate(group=[{0, 2}], agg#0=[AVG($1)]) + LogicalProject(host=[$4], cpu_usage=[$7], $f3=[SPAN($1, 1, 'm')]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + LogicalSort(sort0=[$1], dir0=[DESC], fetch=[10]) + LogicalAggregate(group=[{1}], grand_total=[SUM($2)]) + LogicalFilter(condition=[IS NOT NULL($1)]) + LogicalProject(@timestamp=[$1], host=[$0], $f2=[$2]) + LogicalAggregate(group=[{0, 2}], agg#0=[AVG($1)]) + LogicalProject(host=[$4], cpu_usage=[$7], $f3=[SPAN($1, 1, 'm')]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableAggregate(group=[{0, 1}], avg(cpu_usage)=[SUM($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NOT NULL($t3)], expr#6=[IS NULL($t1)], expr#7=[null:NULL], expr#8=['OTHER'], expr#9=[CASE($t6, $t7, $t8)], expr#10=[CASE($t5, $t1, $t9)], @timestamp=[$t0], host=[$t10], avg(cpu_usage)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], @timestamp=[$t1], host=[$t0], $f2=[$t8]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, cpu_usage, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["host","cpu_usage","@timestamp"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableLimit(fetch=[10]) + EnumerableSort(sort0=[$1], dir0=[DESC]) + EnumerableAggregate(group=[{0}], grand_total=[SUM($1)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], host=[$t0], $f2=[$t8]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, cpu_usage, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"host","boost":1.0}},"_source":{"includes":["host","cpu_usage","@timestamp"],"excludes":[]},"sort":[{"_doc":{"order":"asc"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.json deleted file mode 100644 index 377746b83a5..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC])\n LogicalAggregate(group=[{0, 1}], count=[SUM($2)])\n LogicalUnion(all=[false])\n LogicalAggregate(group=[{0, 1}], actual_count=[SUM($2)])\n LogicalProject(@timestamp=[CAST($0):TIMESTAMP(0) NOT NULL], host=[CASE(IS NOT NULL($3), $1, CASE(IS NULL($1), null:NULL, 'OTHER'))], count=[$2])\n LogicalJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[left])\n LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2])\n LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()])\n LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n LogicalSort(sort0=[$1], dir0=[DESC], fetch=[10])\n LogicalAggregate(group=[{1}], grand_total=[SUM($2)])\n LogicalFilter(condition=[IS NOT NULL($1)])\n LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2])\n LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()])\n LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n LogicalProject(@timestamp=[CAST($0):TIMESTAMP(0) NOT NULL], host=[$1], count=[0])\n LogicalJoin(condition=[true], joinType=[inner])\n LogicalAggregate(group=[{0}])\n LogicalProject(@timestamp=[$1])\n LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()])\n LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n LogicalAggregate(group=[{0}])\n LogicalProject($f0=[CASE(IS NOT NULL($3), $1, CASE(IS NULL($1), null:NULL, 'OTHER'))])\n LogicalJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[left])\n LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2])\n LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()])\n LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n LogicalSort(sort0=[$1], dir0=[DESC], fetch=[10])\n LogicalAggregate(group=[{1}], grand_total=[SUM($2)])\n LogicalFilter(condition=[IS NOT NULL($1)])\n LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2])\n LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()])\n LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC])\n EnumerableAggregate(group=[{0, 1}], count=[$SUM0($2)])\n EnumerableUnion(all=[false])\n EnumerableAggregate(group=[{0, 1}], actual_count=[$SUM0($2)])\n EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t0):TIMESTAMP(0) NOT NULL], expr#6=[IS NOT NULL($t3)], expr#7=[IS NULL($t1)], expr#8=[null:NULL], expr#9=['OTHER'], expr#10=[CASE($t7, $t8, $t9)], expr#11=[CASE($t6, $t1, $t10)], @timestamp=[$t5], host=[$t11], count=[$t2])\n EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left])\n EnumerableSort(sort0=[$1], dir0=[ASC])\n EnumerableCalc(expr#0..2=[{inputs}], @timestamp=[$t1], host=[$t0], $f2_0=[$t2])\n EnumerableAggregate(group=[{0, 1}], agg#0=[COUNT()])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=['m'], expr#4=[SPAN($t1, $t2, $t3)], host=[$t0], $f1=[$t4])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"host\",\"@timestamp\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableLimit(fetch=[10])\n EnumerableSort(sort0=[$1], dir0=[DESC])\n EnumerableAggregate(group=[{0}], grand_total=[COUNT()])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host], FILTER->IS NOT NULL($1), PROJECT->[host, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"host\",\"boost\":1.0}},\"_source\":{\"includes\":[\"host\",\"@timestamp\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):TIMESTAMP(0) NOT NULL], expr#3=[0], @timestamp=[$t2], host=[$t1], count=[$t3])\n EnumerableNestedLoopJoin(condition=[true], joinType=[inner])\n EnumerableAggregate(group=[{1}])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=['m'], expr#4=[SPAN($t1, $t2, $t3)], host=[$t0], $f1=[$t4])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"host\",\"@timestamp\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableAggregate(group=[{0}])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t1)], expr#4=[IS NULL($t0)], expr#5=[null:NULL], expr#6=['OTHER'], expr#7=[CASE($t4, $t5, $t6)], expr#8=[CASE($t3, $t0, $t7)], $f0=[$t8])\n EnumerableMergeJoin(condition=[=($0, $1)], joinType=[left])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableCalc(expr#0..1=[{inputs}], host=[$t0])\n EnumerableAggregate(group=[{0, 1}])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=['m'], expr#4=[SPAN($t1, $t2, $t3)], host=[$t0], $f1=[$t4])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"host\",\"@timestamp\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableLimit(fetch=[10])\n EnumerableSort(sort0=[$1], dir0=[DESC])\n EnumerableAggregate(group=[{0}], grand_total=[COUNT()])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host], FILTER->IS NOT NULL($1), PROJECT->[host, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"host\",\"boost\":1.0}},\"_source\":{\"includes\":[\"host\",\"@timestamp\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml new file mode 100644 index 00000000000..52665f84222 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.yaml @@ -0,0 +1,75 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + LogicalAggregate(group=[{0, 1}], count=[SUM($2)]) + LogicalUnion(all=[false]) + LogicalAggregate(group=[{0, 1}], actual_count=[SUM($2)]) + LogicalProject(@timestamp=[CAST($0):TIMESTAMP(0) NOT NULL], host=[CASE(IS NOT NULL($3), $1, CASE(IS NULL($1), null:NULL, 'OTHER'))], count=[$2]) + LogicalJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[left]) + LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2]) + LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()]) + LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + LogicalSort(sort0=[$1], dir0=[DESC], fetch=[10]) + LogicalAggregate(group=[{1}], grand_total=[SUM($2)]) + LogicalFilter(condition=[IS NOT NULL($1)]) + LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2]) + LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()]) + LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + LogicalProject(@timestamp=[CAST($0):TIMESTAMP(0) NOT NULL], host=[$1], count=[0]) + LogicalJoin(condition=[true], joinType=[inner]) + LogicalAggregate(group=[{0}]) + LogicalProject(@timestamp=[$1]) + LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()]) + LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + LogicalAggregate(group=[{0}]) + LogicalProject($f0=[CASE(IS NOT NULL($3), $1, CASE(IS NULL($1), null:NULL, 'OTHER'))]) + LogicalJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[left]) + LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2]) + LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()]) + LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + LogicalSort(sort0=[$1], dir0=[DESC], fetch=[10]) + LogicalAggregate(group=[{1}], grand_total=[SUM($2)]) + LogicalFilter(condition=[IS NOT NULL($1)]) + LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2]) + LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()]) + LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableAggregate(group=[{0, 1}], count=[$SUM0($2)]) + EnumerableUnion(all=[false]) + EnumerableAggregate(group=[{0, 1}], actual_count=[$SUM0($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t0):TIMESTAMP(0) NOT NULL], expr#6=[IS NOT NULL($t3)], expr#7=[IS NULL($t1)], expr#8=[null:NULL], expr#9=['OTHER'], expr#10=[CASE($t7, $t8, $t9)], expr#11=[CASE($t6, $t1, $t10)], @timestamp=[$t5], host=[$t11], count=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], @timestamp=[$t1], host=[$t0], $f2_0=[$t2]) + EnumerableAggregate(group=[{0, 1}], agg#0=[COUNT()]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=['m'], expr#4=[SPAN($t1, $t2, $t3)], host=[$t0], $f1=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["host","@timestamp"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableLimit(fetch=[10]) + EnumerableSort(sort0=[$1], dir0=[DESC]) + EnumerableAggregate(group=[{0}], grand_total=[COUNT()]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"host","boost":1.0}},"_source":{"includes":["host","@timestamp"],"excludes":[]},"sort":[{"_doc":{"order":"asc"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):TIMESTAMP(0) NOT NULL], expr#3=[0], @timestamp=[$t2], host=[$t1], count=[$t3]) + EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1})], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"$f2":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableAggregate(group=[{0}]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t1)], expr#4=[IS NULL($t0)], expr#5=[null:NULL], expr#6=['OTHER'], expr#7=[CASE($t4, $t5, $t6)], expr#8=[CASE($t3, $t0, $t7)], $f0=[$t8]) + EnumerableMergeJoin(condition=[=($0, $1)], joinType=[left]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..1=[{inputs}], host=[$t0]) + EnumerableAggregate(group=[{0, 1}]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=['m'], expr#4=[SPAN($t1, $t2, $t3)], host=[$t0], $f1=[$t4]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["host","@timestamp"],"excludes":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableLimit(fetch=[10]) + EnumerableSort(sort0=[$1], dir0=[DESC]) + EnumerableAggregate(group=[{0}], grand_total=[COUNT()]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","query":{"exists":{"field":"host","boost":1.0}},"_source":{"includes":["host","@timestamp"],"excludes":[]},"sort":[{"_doc":{"order":"asc"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count_no_pushdown.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count_no_pushdown.json deleted file mode 100644 index 59cb41e32f4..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count_no_pushdown.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC])\n LogicalAggregate(group=[{0, 1}], count=[SUM($2)])\n LogicalUnion(all=[false])\n LogicalAggregate(group=[{0, 1}], actual_count=[SUM($2)])\n LogicalProject(@timestamp=[CAST($0):TIMESTAMP(0) NOT NULL], host=[CASE(IS NOT NULL($3), $1, CASE(IS NULL($1), null:NULL, 'OTHER'))], count=[$2])\n LogicalJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[left])\n LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2])\n LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()])\n LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n LogicalSort(sort0=[$1], dir0=[DESC], fetch=[10])\n LogicalAggregate(group=[{1}], grand_total=[SUM($2)])\n LogicalFilter(condition=[IS NOT NULL($1)])\n LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2])\n LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()])\n LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n LogicalProject(@timestamp=[CAST($0):TIMESTAMP(0) NOT NULL], host=[$1], count=[0])\n LogicalJoin(condition=[true], joinType=[inner])\n LogicalAggregate(group=[{0}])\n LogicalProject(@timestamp=[$1])\n LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()])\n LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n LogicalAggregate(group=[{0}])\n LogicalProject($f0=[CASE(IS NOT NULL($3), $1, CASE(IS NULL($1), null:NULL, 'OTHER'))])\n LogicalJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[left])\n LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2])\n LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()])\n LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n LogicalSort(sort0=[$1], dir0=[DESC], fetch=[10])\n LogicalAggregate(group=[{1}], grand_total=[SUM($2)])\n LogicalFilter(condition=[IS NOT NULL($1)])\n LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2])\n LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()])\n LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC])\n EnumerableAggregate(group=[{0, 1}], count=[$SUM0($2)])\n EnumerableUnion(all=[false])\n EnumerableAggregate(group=[{0, 1}], actual_count=[$SUM0($2)])\n EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t0):TIMESTAMP(0) NOT NULL], expr#6=[IS NOT NULL($t3)], expr#7=[IS NULL($t1)], expr#8=[null:NULL], expr#9=['OTHER'], expr#10=[CASE($t7, $t8, $t9)], expr#11=[CASE($t6, $t1, $t10)], @timestamp=[$t5], host=[$t11], count=[$t2])\n EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left])\n EnumerableSort(sort0=[$1], dir0=[ASC])\n EnumerableCalc(expr#0..2=[{inputs}], @timestamp=[$t1], host=[$t0], $f2_0=[$t2])\n EnumerableAggregate(group=[{0, 1}], agg#0=[COUNT()])\n EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], $f2=[$t18])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableLimit(fetch=[10])\n EnumerableSort(sort0=[$1], dir0=[DESC])\n EnumerableAggregate(group=[{4}], grand_total=[COUNT()])\n EnumerableCalc(expr#0..15=[{inputs}], expr#16=[IS NOT NULL($t4)], proj#0..15=[{exprs}], $condition=[$t16])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):TIMESTAMP(0) NOT NULL], expr#3=[0], @timestamp=[$t2], host=[$t1], count=[$t3])\n EnumerableNestedLoopJoin(condition=[true], joinType=[inner])\n EnumerableAggregate(group=[{1}])\n EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], $f2=[$t18])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]])\n EnumerableAggregate(group=[{0}])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t1)], expr#4=[IS NULL($t0)], expr#5=[null:NULL], expr#6=['OTHER'], expr#7=[CASE($t4, $t5, $t6)], expr#8=[CASE($t3, $t0, $t7)], $f0=[$t8])\n EnumerableMergeJoin(condition=[=($0, $1)], joinType=[left])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableCalc(expr#0..1=[{inputs}], host=[$t0])\n EnumerableAggregate(group=[{0, 1}])\n EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], $f2=[$t18])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableLimit(fetch=[10])\n EnumerableSort(sort0=[$1], dir0=[DESC])\n EnumerableAggregate(group=[{4}], grand_total=[COUNT()])\n EnumerableCalc(expr#0..15=[{inputs}], expr#16=[IS NOT NULL($t4)], proj#0..15=[{exprs}], $condition=[$t16])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]])\n"}} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count_no_pushdown.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count_no_pushdown.yaml new file mode 100644 index 00000000000..02f61d2177c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count_no_pushdown.yaml @@ -0,0 +1,79 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + LogicalAggregate(group=[{0, 1}], count=[SUM($2)]) + LogicalUnion(all=[false]) + LogicalAggregate(group=[{0, 1}], actual_count=[SUM($2)]) + LogicalProject(@timestamp=[CAST($0):TIMESTAMP(0) NOT NULL], host=[CASE(IS NOT NULL($3), $1, CASE(IS NULL($1), null:NULL, 'OTHER'))], count=[$2]) + LogicalJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[left]) + LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2]) + LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()]) + LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + LogicalSort(sort0=[$1], dir0=[DESC], fetch=[10]) + LogicalAggregate(group=[{1}], grand_total=[SUM($2)]) + LogicalFilter(condition=[IS NOT NULL($1)]) + LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2]) + LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()]) + LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + LogicalProject(@timestamp=[CAST($0):TIMESTAMP(0) NOT NULL], host=[$1], count=[0]) + LogicalJoin(condition=[true], joinType=[inner]) + LogicalAggregate(group=[{0}]) + LogicalProject(@timestamp=[$1]) + LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()]) + LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + LogicalAggregate(group=[{0}]) + LogicalProject($f0=[CASE(IS NOT NULL($3), $1, CASE(IS NULL($1), null:NULL, 'OTHER'))]) + LogicalJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[left]) + LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2]) + LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()]) + LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + LogicalSort(sort0=[$1], dir0=[DESC], fetch=[10]) + LogicalAggregate(group=[{1}], grand_total=[SUM($2)]) + LogicalFilter(condition=[IS NOT NULL($1)]) + LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2]) + LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()]) + LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableAggregate(group=[{0, 1}], count=[$SUM0($2)]) + EnumerableUnion(all=[false]) + EnumerableAggregate(group=[{0, 1}], actual_count=[$SUM0($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t0):TIMESTAMP(0) NOT NULL], expr#6=[IS NOT NULL($t3)], expr#7=[IS NULL($t1)], expr#8=[null:NULL], expr#9=['OTHER'], expr#10=[CASE($t7, $t8, $t9)], expr#11=[CASE($t6, $t1, $t10)], @timestamp=[$t5], host=[$t11], count=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..2=[{inputs}], @timestamp=[$t1], host=[$t0], $f2_0=[$t2]) + EnumerableAggregate(group=[{0, 1}], agg#0=[COUNT()]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], $f2=[$t18]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableLimit(fetch=[10]) + EnumerableSort(sort0=[$1], dir0=[DESC]) + EnumerableAggregate(group=[{4}], grand_total=[COUNT()]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[IS NOT NULL($t4)], proj#0..15=[{exprs}], $condition=[$t16]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):TIMESTAMP(0) NOT NULL], expr#3=[0], @timestamp=[$t2], host=[$t1], count=[$t3]) + EnumerableNestedLoopJoin(condition=[true], joinType=[inner]) + EnumerableAggregate(group=[{1}]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], $f2=[$t18]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) + EnumerableAggregate(group=[{0}]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t1)], expr#4=[IS NULL($t0)], expr#5=[null:NULL], expr#6=['OTHER'], expr#7=[CASE($t4, $t5, $t6)], expr#8=[CASE($t3, $t0, $t7)], $f0=[$t8]) + EnumerableMergeJoin(condition=[=($0, $1)], joinType=[left]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..1=[{inputs}], host=[$t0]) + EnumerableAggregate(group=[{0, 1}]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], $f2=[$t18]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableLimit(fetch=[10]) + EnumerableSort(sort0=[$1], dir0=[DESC]) + EnumerableAggregate(group=[{4}], grand_total=[COUNT()]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[IS NOT NULL($t4)], proj#0..15=[{exprs}], $condition=[$t16]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_no_pushdown.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_no_pushdown.json deleted file mode 100644 index cfb2a71a690..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_no_pushdown.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC])\n LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[SUM($2)])\n LogicalProject(@timestamp=[$0], host=[CASE(IS NOT NULL($3), $1, CASE(IS NULL($1), null:NULL, 'OTHER'))], avg(cpu_usage)=[$2])\n LogicalJoin(condition=[=($1, $3)], joinType=[left])\n LogicalProject(@timestamp=[$1], host=[$0], $f2=[$2])\n LogicalAggregate(group=[{0, 2}], agg#0=[AVG($1)])\n LogicalProject(host=[$4], cpu_usage=[$7], $f3=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n LogicalSort(sort0=[$1], dir0=[DESC], fetch=[10])\n LogicalAggregate(group=[{1}], grand_total=[SUM($2)])\n LogicalFilter(condition=[IS NOT NULL($1)])\n LogicalProject(@timestamp=[$1], host=[$0], $f2=[$2])\n LogicalAggregate(group=[{0, 2}], agg#0=[AVG($1)])\n LogicalProject(host=[$4], cpu_usage=[$7], $f3=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC])\n EnumerableAggregate(group=[{0, 1}], avg(cpu_usage)=[SUM($2)])\n EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NOT NULL($t3)], expr#6=[IS NULL($t1)], expr#7=[null:NULL], expr#8=['OTHER'], expr#9=[CASE($t6, $t7, $t8)], expr#10=[CASE($t5, $t1, $t9)], @timestamp=[$t0], host=[$t10], avg(cpu_usage)=[$t2])\n EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left])\n EnumerableSort(sort0=[$1], dir0=[ASC])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], @timestamp=[$t1], host=[$t0], $f2=[$t8])\n EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)])\n EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], cpu_usage=[$t7], $f3=[$t18])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableLimit(fetch=[10])\n EnumerableSort(sort0=[$1], dir0=[DESC])\n EnumerableAggregate(group=[{0}], grand_total=[SUM($2)])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], expr#9=[IS NOT NULL($t0)], proj#0..1=[{exprs}], $f2=[$t8], $condition=[$t9])\n EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)])\n EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], cpu_usage=[$t7], $f3=[$t18])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_no_pushdown.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_no_pushdown.yaml new file mode 100644 index 00000000000..ae966d7eea7 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_no_pushdown.yaml @@ -0,0 +1,37 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[SUM($2)]) + LogicalProject(@timestamp=[$0], host=[CASE(IS NOT NULL($3), $1, CASE(IS NULL($1), null:NULL, 'OTHER'))], avg(cpu_usage)=[$2]) + LogicalJoin(condition=[=($1, $3)], joinType=[left]) + LogicalProject(@timestamp=[$1], host=[$0], $f2=[$2]) + LogicalAggregate(group=[{0, 2}], agg#0=[AVG($1)]) + LogicalProject(host=[$4], cpu_usage=[$7], $f3=[SPAN($1, 1, 'm')]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + LogicalSort(sort0=[$1], dir0=[DESC], fetch=[10]) + LogicalAggregate(group=[{1}], grand_total=[SUM($2)]) + LogicalFilter(condition=[IS NOT NULL($1)]) + LogicalProject(@timestamp=[$1], host=[$0], $f2=[$2]) + LogicalAggregate(group=[{0, 2}], agg#0=[AVG($1)]) + LogicalProject(host=[$4], cpu_usage=[$7], $f3=[SPAN($1, 1, 'm')]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC]) + EnumerableAggregate(group=[{0, 1}], avg(cpu_usage)=[SUM($2)]) + EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NOT NULL($t3)], expr#6=[IS NULL($t1)], expr#7=[null:NULL], expr#8=['OTHER'], expr#9=[CASE($t6, $t7, $t8)], expr#10=[CASE($t5, $t1, $t9)], @timestamp=[$t0], host=[$t10], avg(cpu_usage)=[$t2]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], @timestamp=[$t1], host=[$t0], $f2=[$t8]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], cpu_usage=[$t7], $f3=[$t18]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableLimit(fetch=[10]) + EnumerableSort(sort0=[$1], dir0=[DESC]) + EnumerableAggregate(group=[{0}], grand_total=[SUM($2)]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], expr#9=[IS NOT NULL($t0)], proj#0..1=[{exprs}], $f2=[$t8], $condition=[$t9]) + EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)]) + EnumerableCalc(expr#0..15=[{inputs}], expr#16=[1], expr#17=['m'], expr#18=[SPAN($t1, $t16, $t17)], host=[$t4], cpu_usage=[$t7], $f3=[$t18]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_push.json deleted file mode 100644 index 2444b20b857..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(ageTrend=[CASE(>(COUNT() OVER (ROWS 1 PRECEDING), 1), /(SUM($8) OVER (ROWS 1 PRECEDING), CAST(COUNT($8) OVER (ROWS 1 PRECEDING)):DOUBLE NOT NULL), null:NULL)])\n LogicalFilter(condition=[IS NOT NULL($8)])\n LogicalSort(fetch=[5])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[>($t1, $t4)], expr#6=[CAST($t3):DOUBLE NOT NULL], expr#7=[/($t2, $t6)], expr#8=[null:NULL], expr#9=[CASE($t5, $t7, $t8)], ageTrend=[$t9])\n EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), $SUM0($0), COUNT($0)])])\n EnumerableCalc(expr#0=[{inputs}], expr#1=[IS NOT NULL($t0)], age=[$t0], $condition=[$t1])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[LIMIT->5, PROJECT->[age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_push.yaml new file mode 100644 index 00000000000..fc9c57db158 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(ageTrend=[CASE(>(COUNT() OVER (ROWS 1 PRECEDING), 1), /(SUM($8) OVER (ROWS 1 PRECEDING), CAST(COUNT($8) OVER (ROWS 1 PRECEDING)):DOUBLE NOT NULL), null:NULL)]) + LogicalFilter(condition=[IS NOT NULL($8)]) + LogicalSort(fetch=[5]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[>($t1, $t4)], expr#6=[CAST($t3):DOUBLE NOT NULL], expr#7=[/($t2, $t6)], expr#8=[null:NULL], expr#9=[CASE($t5, $t7, $t8)], ageTrend=[$t9]) + EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), $SUM0($0), COUNT($0)])]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[IS NOT NULL($t0)], age=[$t0], $condition=[$t1]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_sort_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_sort_push.json deleted file mode 100644 index 1f2a6dcfa16..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_sort_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(ageTrend=[CASE(>(COUNT() OVER (ROWS 1 PRECEDING), 1), /(SUM($8) OVER (ROWS 1 PRECEDING), CAST(COUNT($8) OVER (ROWS 1 PRECEDING)):DOUBLE NOT NULL), null:NULL)])\n LogicalFilter(condition=[IS NOT NULL($8)])\n LogicalSort(sort0=[$8], dir0=[ASC])\n LogicalSort(fetch=[5])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[>($t1, $t4)], expr#6=[CAST($t3):DOUBLE NOT NULL], expr#7=[/($t2, $t6)], expr#8=[null:NULL], expr#9=[CASE($t5, $t7, $t8)], ageTrend=[$t9])\n EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), $SUM0($0), COUNT($0)])])\n EnumerableCalc(expr#0=[{inputs}], expr#1=[IS NOT NULL($t0)], age=[$t0], $condition=[$t1])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[LIMIT->5, PROJECT->[age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_sort_push.yaml new file mode 100644 index 00000000000..2003143e673 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_trendline_sort_push.yaml @@ -0,0 +1,15 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(ageTrend=[CASE(>(COUNT() OVER (ROWS 1 PRECEDING), 1), /(SUM($8) OVER (ROWS 1 PRECEDING), CAST(COUNT($8) OVER (ROWS 1 PRECEDING)):DOUBLE NOT NULL), null:NULL)]) + LogicalFilter(condition=[IS NOT NULL($8)]) + LogicalSort(sort0=[$8], dir0=[ASC]) + LogicalSort(fetch=[5]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[>($t1, $t4)], expr#6=[CAST($t3):DOUBLE NOT NULL], expr#7=[/($t2, $t6)], expr#8=[null:NULL], expr#9=[CASE($t5, $t7, $t8)], ageTrend=[$t9]) + EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), $SUM0($0), COUNT($0)])]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[IS NOT NULL($t0)], age=[$t0], $condition=[$t1]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], LIMIT->5], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["age"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.json deleted file mode 100644 index 90ae3beb4d2..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(sum(balance)=[$1], sum(balance + 100)=[$2], sum(balance - 100)=[$3], sum(balance * 100)=[$4], sum(balance / 100)=[$5], gender=[$0])\n LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)=[SUM($2)], sum(balance - 100)=[SUM($3)], sum(balance * 100)=[SUM($4)], sum(balance / 100)=[SUM($5)])\n LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0])\n EnumerableAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)_COUNT=[COUNT($1)], sum(balance / 100)=[SUM($2)])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[100], expr#20=[DIVIDE($t7, $t19)], gender=[$t4], balance=[$t7], $f5=[$t20])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml new file mode 100644 index 00000000000..bf861c337b9 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(sum(balance)=[$1], sum(balance + 100)=[$2], sum(balance - 100)=[$3], sum(balance * 100)=[$4], sum(balance / 100)=[$5], gender=[$0]) + LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)=[SUM($2)], sum(balance - 100)=[SUM($3)], sum(balance * 100)=[SUM($4)], sum(balance / 100)=[SUM($5)]) + LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) + EnumerableAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)_COUNT=[COUNT($1)], sum(balance / 100)=[SUM($2)]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[100], expr#20=[DIVIDE($t7, $t19)], gender=[$t4], balance=[$t7], $f5=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_aligntime.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_aligntime.json deleted file mode 100644 index 8b1804a42a3..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_aligntime.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(category=[$0], value=[$1], timestamp=[$2], @timestamp=[$9])\n LogicalSort(fetch=[5])\n LogicalProject(category=[$1], value=[$2], timestamp=[$3], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9], @timestamp=[FROM_UNIXTIME(*(*(FLOOR(/(/(UNIX_TIMESTAMP($0), 3600), 2)), 2), 3600))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..9=[{inputs}], expr#10=[UNIX_TIMESTAMP($t0)], expr#11=[3600], expr#12=[/($t10, $t11)], expr#13=[2], expr#14=[/($t12, $t13)], expr#15=[FLOOR($t14)], expr#16=[*($t15, $t13)], expr#17=[*($t16, $t11)], expr#18=[FROM_UNIXTIME($t17)], category=[$t1], value=[$t2], timestamp=[$t3], @timestamp=[$t18])\n EnumerableLimit(fetch=[5])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]])\n"}} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_aligntime.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_aligntime.yaml new file mode 100644 index 00000000000..282a080733e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_aligntime.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(category=[$0], value=[$1], timestamp=[$2], @timestamp=[$9]) + LogicalSort(fetch=[5]) + LogicalProject(category=[$1], value=[$2], timestamp=[$3], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9], @timestamp=[FROM_UNIXTIME(*(*(FLOOR(/(/(UNIX_TIMESTAMP($0), 3600), 2)), 2), 3600))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..9=[{inputs}], expr#10=[UNIX_TIMESTAMP($t0)], expr#11=[3600], expr#12=[/($t10, $t11)], expr#13=[2], expr#14=[/($t12, $t13)], expr#15=[FLOOR($t14)], expr#16=[*($t15, $t13)], expr#17=[*($t16, $t11)], expr#18=[FROM_UNIXTIME($t17)], category=[$t1], value=[$t2], timestamp=[$t3], @timestamp=[$t18]) + EnumerableLimit(fetch=[5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_span.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_span.json deleted file mode 100644 index bacc2e0fca3..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_span.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[SPAN_BUCKET($8, 10)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[10], expr#18=[SPAN_BUCKET($t8, $t17)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t18])\n EnumerableLimit(fetch=[5])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n"}} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_span.yaml new file mode 100644 index 00000000000..13a1b2e5029 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_bin_span.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$8], lastname=[$9], age=[$16]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], age=[SPAN_BUCKET($8, 10)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[10], expr#18=[SPAN_BUCKET($t8, $t17)], proj#0..7=[{exprs}], email=[$t9], lastname=[$t10], age=[$t18]) + EnumerableLimit(fetch=[5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.json deleted file mode 100644 index 7008e2fa0dc..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age=[$8])\n LogicalFilter(condition=[>($8, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_age=[$t9], state=[$t1], city=[$t0])\n EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml new file mode 100644 index 00000000000..ac3728eacb9 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_agg_push.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg_age=[$2], state=[$0], city=[$1]) + LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) + LogicalProject(state=[$7], city=[$5], age=[$8]) + LogicalFilter(condition=[>($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], avg_age=[$t9], state=[$t1], city=[$t0]) + EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push.json deleted file mode 100644 index 3e9d3b3df01..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalFilter(condition=[>($3, 10000)])\n LogicalFilter(condition=[<($8, 40)])\n LogicalFilter(condition=[>($8, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[Sarg[(30..40)]], expr#18=[SEARCH($t8, $t17)], expr#19=[10000], expr#20=[>($t3, $t19)], expr#21=[AND($t18, $t20)], age=[$t8], $condition=[$t21])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push.yaml new file mode 100644 index 00000000000..ff9e2ed0ec1 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalFilter(condition=[>($3, 10000)]) + LogicalFilter(condition=[<($8, 40)]) + LogicalFilter(condition=[>($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[Sarg[(30..40)]], expr#18=[SEARCH($t8, $t17)], expr#19=[10000], expr#20=[>($t3, $t19)], expr#21=[AND($t18, $t20)], age=[$t8], $condition=[$t21]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_date_string.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_date_string.json deleted file mode 100644 index ec9d500b4d1..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_date_string.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalFilter(condition=[<($0, DATE('2018-11-09 00:00:00.000000000':VARCHAR))])\n LogicalFilter(condition=[>($0, DATE('2016-12-08 00:00:00.123456789':VARCHAR))])\n LogicalProject(yyyy-MM-dd=[$83])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..94=[{inputs}], expr#95=[Sarg[('2016-12-08':VARCHAR..'2018-11-09':VARCHAR)]:VARCHAR], expr#96=[SEARCH($t83, $t95)], yyyy-MM-dd=[$t83], $condition=[$t96])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_date_string.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_date_string.yaml new file mode 100644 index 00000000000..a8f52a8ac7e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_date_string.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalFilter(condition=[<($0, DATE('2018-11-09 00:00:00.000000000':VARCHAR))]) + LogicalFilter(condition=[>($0, DATE('2016-12-08 00:00:00.123456789':VARCHAR))]) + LogicalProject(yyyy-MM-dd=[$83]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..94=[{inputs}], expr#95=[Sarg[('2016-12-08':VARCHAR..'2018-11-09':VARCHAR)]:VARCHAR], expr#96=[SEARCH($t83, $t95)], yyyy-MM-dd=[$t83], $condition=[$t96]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_time_string.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_time_string.json deleted file mode 100644 index 7118b4ac807..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_time_string.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalFilter(condition=[<($0, TIME('2018-11-09 19:00:00.123456789':VARCHAR))])\n LogicalFilter(condition=[>($0, TIME('2016-12-08 12:00:00.123456789':VARCHAR))])\n LogicalProject(custom_time=[$49])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..94=[{inputs}], expr#95=[Sarg[('12:00:00.123456789':VARCHAR..'19:00:00.123456789':VARCHAR)]:VARCHAR], expr#96=[SEARCH($t49, $t95)], custom_time=[$t49], $condition=[$t96])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_time_string.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_time_string.yaml new file mode 100644 index 00000000000..72c738eaed0 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_time_string.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalFilter(condition=[<($0, TIME('2018-11-09 19:00:00.123456789':VARCHAR))]) + LogicalFilter(condition=[>($0, TIME('2016-12-08 12:00:00.123456789':VARCHAR))]) + LogicalProject(custom_time=[$49]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..94=[{inputs}], expr#95=[Sarg[('12:00:00.123456789':VARCHAR..'19:00:00.123456789':VARCHAR)]:VARCHAR], expr#96=[SEARCH($t49, $t95)], custom_time=[$t49], $condition=[$t96]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_date_formats]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_timestamp_string.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_timestamp_string.json deleted file mode 100644 index 7540c6dd7a5..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_timestamp_string.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12])\n LogicalFilter(condition=[<($3, TIMESTAMP('2018-11-09 00:00:00.000000000':VARCHAR))])\n LogicalFilter(condition=[>($3, TIMESTAMP('2016-12-08 00:00:00.000000000':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[Sarg[('2016-12-08 00:00:00':VARCHAR..'2018-11-09 00:00:00':VARCHAR)]:VARCHAR], expr#20=[SEARCH($t3, $t19)], proj#0..12=[{exprs}], $condition=[$t20])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_timestamp_string.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_timestamp_string.yaml new file mode 100644 index 00000000000..424444f8dc4 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_push_compare_timestamp_string.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + LogicalFilter(condition=[<($3, TIMESTAMP('2018-11-09 00:00:00.000000000':VARCHAR))]) + LogicalFilter(condition=[>($3, TIMESTAMP('2016-12-08 00:00:00.000000000':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[Sarg[('2016-12-08 00:00:00':VARCHAR..'2018-11-09 00:00:00':VARCHAR)]:VARCHAR], expr#20=[SEARCH($t3, $t19)], proj#0..12=[{exprs}], $condition=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_then_limit_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_then_limit_push.json deleted file mode 100644 index 24123421480..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_then_limit_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalSort(fetch=[5])\n LogicalFilter(condition=[>($8, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], age=[$t8])\n EnumerableLimit(fetch=[5])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_then_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_then_limit_push.yaml new file mode 100644 index 00000000000..8470ec7d765 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_then_limit_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalSort(fetch=[5]) + LogicalFilter(condition=[>($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], age=[$t8]) + EnumerableLimit(fetch=[5]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10_5_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10_5_push.json deleted file mode 100644 index b6611fbb093..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10_5_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalSort(fetch=[5])\n LogicalSort(fetch=[10])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], age=[$t8])\n EnumerableLimit(fetch=[5])\n EnumerableLimit(fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10_5_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10_5_push.yaml new file mode 100644 index 00000000000..ebdf44514de --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10_5_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalSort(fetch=[5]) + LogicalSort(fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], age=[$t8]) + EnumerableLimit(fetch=[5]) + EnumerableLimit(fetch=[10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10_filter_5_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10_filter_5_push.json deleted file mode 100644 index e8d337dbf13..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10_filter_5_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalSort(fetch=[5])\n LogicalFilter(condition=[>($8, 30)])\n LogicalSort(fetch=[10])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableLimit(fetch=[5])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], age=[$t8], $condition=[$t18])\n EnumerableLimit(fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10_filter_5_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10_filter_5_push.yaml new file mode 100644 index 00000000000..54bf835b3ef --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10_filter_5_push.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalSort(fetch=[5]) + LogicalFilter(condition=[>($8, 30)]) + LogicalSort(fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(fetch=[5]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], age=[$t8], $condition=[$t18]) + EnumerableLimit(fetch=[10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10from1_10from2_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10from1_10from2_push.json deleted file mode 100644 index ea9965cd5c5..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10from1_10from2_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalSort(offset=[2], fetch=[10])\n LogicalSort(offset=[1], fetch=[10])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], age=[$t8])\n EnumerableLimit(offset=[2], fetch=[10])\n EnumerableLimit(offset=[1], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10from1_10from2_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10from1_10from2_push.yaml new file mode 100644 index 00000000000..544f0ac88bb --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_10from1_10from2_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalSort(offset=[2], fetch=[10]) + LogicalSort(offset=[1], fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], age=[$t8]) + EnumerableLimit(offset=[2], fetch=[10]) + EnumerableLimit(offset=[1], fetch=[10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_5_10_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_5_10_push.json deleted file mode 100644 index 3f51103c093..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_5_10_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalSort(fetch=[10])\n LogicalSort(fetch=[5])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableLimit(fetch=[10])\n EnumerableCalc(expr#0..16=[{inputs}], age=[$t8])\n EnumerableLimit(fetch=[5])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_5_10_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_5_10_push.yaml new file mode 100644 index 00000000000..17801ae089a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_5_10_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalSort(fetch=[10]) + LogicalSort(fetch=[5]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(fetch=[10]) + EnumerableCalc(expr#0..16=[{inputs}], age=[$t8]) + EnumerableLimit(fetch=[5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_offsets_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_offsets_push.json deleted file mode 100644 index 872e87437da..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_offsets_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalSort(offset=[2], fetch=[5])\n LogicalSort(offset=[1], fetch=[10])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], age=[$t8])\n EnumerableLimit(offset=[2], fetch=[5])\n EnumerableLimit(offset=[1], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_offsets_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_offsets_push.yaml new file mode 100644 index 00000000000..6f901ac6276 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_offsets_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalSort(offset=[2], fetch=[5]) + LogicalSort(offset=[1], fetch=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], age=[$t8]) + EnumerableLimit(offset=[2], fetch=[5]) + EnumerableLimit(offset=[1], fetch=[10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.json deleted file mode 100644 index 22b72198680..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(ageMinus=[$17])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], ageMinus=[-($8, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[-($t8, $t17)], ageMinus=[$t18])\n EnumerableLimit(fetch=[5])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.yaml new file mode 100644 index 00000000000..fb3daa06769 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_push.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(ageMinus=[$17]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], ageMinus=[-($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[-($t8, $t17)], ageMinus=[$t18]) + EnumerableLimit(fetch=[5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_then_filter_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_then_filter_push.json deleted file mode 100644 index 38d252d12c0..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_then_filter_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalFilter(condition=[>($8, 30)])\n LogicalSort(fetch=[5])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], age=[$t8], $condition=[$t18])\n EnumerableLimit(fetch=[5])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_then_filter_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_then_filter_push.yaml new file mode 100644 index 00000000000..6bae1a225fb --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_then_filter_push.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalFilter(condition=[>($8, 30)]) + LogicalSort(fetch=[5]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], age=[$t8], $condition=[$t18]) + EnumerableLimit(fetch=[5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_then_sort_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_then_sort_push.json deleted file mode 100644 index ece4df73ede..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_then_sort_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalSort(sort0=[$8], dir0=[ASC-nulls-first])\n LogicalSort(fetch=[5])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..16=[{inputs}], age=[$t8])\n EnumerableLimit(fetch=[5])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_then_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_then_sort_push.yaml new file mode 100644 index 00000000000..9b7aa3c87b0 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_limit_then_sort_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalSort(sort0=[$8], dir0=[ASC-nulls-first]) + LogicalSort(fetch=[5]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first]) + EnumerableCalc(expr#0..16=[{inputs}], age=[$t8]) + EnumerableLimit(fetch=[5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.json deleted file mode 100644 index 6b8e10d07b3..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age2=[$2])\n LogicalFilter(condition=[<=($3, 1)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2 ORDER BY $2)])\n LogicalFilter(condition=[IS NOT NULL($2)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age=[$8])\n LogicalFilter(condition=[>($8, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableCalc(expr#0..2=[{inputs}], age2=[$t1])\n EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4])\n EnumerableWindow(window#0=[window(partition {1} order by [1] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[2], expr#11=[+($t9, $t10)], expr#12=[IS NOT NULL($t8)], state=[$t1], age2=[$t11], $condition=[$t12])\n EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first])\n EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml new file mode 100644 index 00000000000..8c0283f0e9c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_output.yaml @@ -0,0 +1,24 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age2=[$2]) + LogicalFilter(condition=[<=($3, 1)]) + LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2 ORDER BY $2)]) + LogicalFilter(condition=[IS NOT NULL($2)]) + LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)]) + LogicalSort(sort0=[$1], dir0=[ASC-nulls-first]) + LogicalProject(avg_age=[$2], state=[$0], city=[$1]) + LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)]) + LogicalProject(state=[$7], city=[$5], age=[$8]) + LogicalFilter(condition=[>($8, 30)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..2=[{inputs}], age2=[$t1]) + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) + EnumerableWindow(window#0=[window(partition {1} order by [1] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:BIGINT], expr#7=[CASE($t5, $t6, $t2)], expr#8=[CAST($t7):DOUBLE], expr#9=[/($t8, $t3)], expr#10=[2], expr#11=[+($t9, $t10)], expr#12=[IS NOT NULL($t8)], state=[$t1], age2=[$t11], $condition=[$t12]) + EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first]) + EnumerableAggregate(group=[{5, 7}], agg#0=[$SUM0($8)], agg#1=[COUNT($8)]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[30], expr#18=[>($t8, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_simple_pattern_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_simple_pattern_agg_push.json deleted file mode 100644 index 73affef775a..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_simple_pattern_agg_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(patterns_field=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'pattern'))], pattern_count=[$1], tokens=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'tokens'))], sample_logs=[$2])\n LogicalAggregate(group=[{1}], pattern_count=[COUNT($1)], sample_logs=[TAKE($0, $2)])\n LogicalProject(email=[$9], patterns_field=[REGEXP_REPLACE($9, '[a-zA-Z0-9]+':VARCHAR, '<*>')], $f18=[10])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[PATTERN_PARSER($t0, $t2)], expr#4=['pattern'], expr#5=[ITEM($t3, $t4)], expr#6=[SAFE_CAST($t5)], expr#7=['tokens'], expr#8=[ITEM($t3, $t7)], expr#9=[SAFE_CAST($t8)], patterns_field=[$t6], pattern_count=[$t1], tokens=[$t9], sample_logs=[$t2])\n EnumerableAggregate(group=[{1}], pattern_count=[COUNT($1)], sample_logs=[TAKE($0, $2)])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=['[a-zA-Z0-9]+':VARCHAR], expr#18=['<*>'], expr#19=[REGEXP_REPLACE($t9, $t17, $t18)], expr#20=[10], email=[$t9], patterns_field=[$t19], $f18=[$t20])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_simple_pattern_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_simple_pattern_agg_push.yaml new file mode 100644 index 00000000000..f9d28c5503b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_simple_pattern_agg_push.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(patterns_field=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'pattern'))], pattern_count=[$1], tokens=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'tokens'))], sample_logs=[$2]) + LogicalAggregate(group=[{1}], pattern_count=[COUNT($1)], sample_logs=[TAKE($0, $2)]) + LogicalProject(email=[$9], patterns_field=[REGEXP_REPLACE($9, '[a-zA-Z0-9]+':VARCHAR, '<*>')], $f18=[10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[PATTERN_PARSER($t0, $t2)], expr#4=['pattern'], expr#5=[ITEM($t3, $t4)], expr#6=[SAFE_CAST($t5)], expr#7=['tokens'], expr#8=[ITEM($t3, $t7)], expr#9=[SAFE_CAST($t8)], patterns_field=[$t6], pattern_count=[$t1], tokens=[$t9], sample_logs=[$t2]) + EnumerableAggregate(group=[{1}], pattern_count=[COUNT($1)], sample_logs=[TAKE($0, $2)]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['[a-zA-Z0-9]+':VARCHAR], expr#18=['<*>'], expr#19=[REGEXP_REPLACE($t9, $t17, $t18)], expr#20=[10], email=[$t9], patterns_field=[$t19], $f18=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regex.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regex.json deleted file mode 100644 index 1b72a2e91ff..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regex.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10])\n LogicalSort(fetch=[5])\n LogicalFilter(condition=[REGEXP_CONTAINS($10, '^[A-Z][a-z]+$':VARCHAR)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}])\n EnumerableLimit(fetch=[5])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=['^[A-Z][a-z]+$':VARCHAR], expr#18=[REGEXP_CONTAINS($t10, $t17)], proj#0..16=[{exprs}], $condition=[$t18])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n"}} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regex.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regex.yaml new file mode 100644 index 00000000000..1584ec71f53 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regex.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalSort(fetch=[5]) + LogicalFilter(condition=[REGEXP_CONTAINS($10, '^[A-Z][a-z]+$':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}]) + EnumerableLimit(fetch=[5]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['^[A-Z][a-z]+$':VARCHAR], expr#18=[REGEXP_CONTAINS($t10, $t17)], proj#0..16=[{exprs}], $condition=[$t18]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regex_negated.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regex_negated.json deleted file mode 100644 index 85a3071c029..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regex_negated.json +++ /dev/null @@ -1 +0,0 @@ -{"calcite":{"logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10])\n LogicalSort(fetch=[5])\n LogicalFilter(condition=[NOT(REGEXP_CONTAINS($10, '.*son$':VARCHAR))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}])\n EnumerableLimit(fetch=[5])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=['.*son$':VARCHAR], expr#18=[REGEXP_CONTAINS($t10, $t17)], expr#19=[NOT($t18)], proj#0..16=[{exprs}], $condition=[$t19])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n"}} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regex_negated.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regex_negated.yaml new file mode 100644 index 00000000000..b16c26d4169 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_regex_negated.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10]) + LogicalSort(fetch=[5]) + LogicalFilter(condition=[NOT(REGEXP_CONTAINS($10, '.*son$':VARCHAR))]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], proj#0..10=[{exprs}]) + EnumerableLimit(fetch=[5]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['.*son$':VARCHAR], expr#18=[REGEXP_CONTAINS($t10, $t17)], expr#19=[NOT($t18)], proj#0..16=[{exprs}], $condition=[$t19]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rex.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rex.json deleted file mode 100644 index a64e2edeea2..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rex.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], initial=[$17])\n LogicalSort(fetch=[5])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], initial=[REX_EXTRACT($10, '(?^[A-Z])', 1)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=['(?^[A-Z])'], expr#18=[1], expr#19=[REX_EXTRACT($t10, $t17, $t18)], proj#0..10=[{exprs}], initial=[$t19])\n EnumerableLimit(fetch=[5])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rex.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rex.yaml new file mode 100644 index 00000000000..00dc5e4ef91 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_rex.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], initial=[$17]) + LogicalSort(fetch=[5]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], initial=[REX_EXTRACT($10, '(?^[A-Z])', 1)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=['(?^[A-Z])'], expr#18=[1], expr#19=[REX_EXTRACT($t10, $t17, $t18)], proj#0..10=[{exprs}], initial=[$t19]) + EnumerableLimit(fetch=[5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_count_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_count_push.json deleted file mode 100644 index 4a1f1510874..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_count_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalSort(sort0=[$8], dir0=[ASC-nulls-first], fetch=[5])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], age=[$t8])\n EnumerableLimit(fetch=[5])\n EnumerableSort(sort0=[$8], dir0=[ASC-nulls-first])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_count_push.yaml new file mode 100644 index 00000000000..c39d769eb4b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_count_push.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalSort(sort0=[$8], dir0=[ASC-nulls-first], fetch=[5]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], age=[$t8]) + EnumerableLimit(fetch=[5]) + EnumerableSort(sort0=[$8], dir0=[ASC-nulls-first]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_limit_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_limit_push.json deleted file mode 100644 index 200c29ea3c5..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_limit_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalSort(sort0=[$8], dir0=[ASC-nulls-first], fetch=[5])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..16=[{inputs}], age=[$t8])\n EnumerableLimit(fetch=[5])\n EnumerableSort(sort0=[$8], dir0=[ASC-nulls-first])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_limit_push.yaml new file mode 100644 index 00000000000..c39d769eb4b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_then_limit_push.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(age=[$8]) + LogicalSort(sort0=[$8], dir0=[ASC-nulls-first], fetch=[5]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..16=[{inputs}], age=[$t8]) + EnumerableLimit(fetch=[5]) + EnumerableSort(sort0=[$8], dir0=[ASC-nulls-first]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_push.json deleted file mode 100644 index 5065eb94b53..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(ageTrend=[CASE(>(COUNT() OVER (ROWS 1 PRECEDING), 1), /(SUM($8) OVER (ROWS 1 PRECEDING), CAST(COUNT($8) OVER (ROWS 1 PRECEDING)):DOUBLE NOT NULL), null:NULL)])\n LogicalFilter(condition=[IS NOT NULL($8)])\n LogicalSort(fetch=[5])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[>($t1, $t4)], expr#6=[CAST($t3):DOUBLE NOT NULL], expr#7=[/($t2, $t6)], expr#8=[null:NULL], expr#9=[CASE($t5, $t7, $t8)], ageTrend=[$t9])\n EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), $SUM0($0), COUNT($0)])])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NOT NULL($t8)], age=[$t8], $condition=[$t17])\n EnumerableLimit(fetch=[5])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_push.yaml new file mode 100644 index 00000000000..74d2b3d4d49 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_push.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(ageTrend=[CASE(>(COUNT() OVER (ROWS 1 PRECEDING), 1), /(SUM($8) OVER (ROWS 1 PRECEDING), CAST(COUNT($8) OVER (ROWS 1 PRECEDING)):DOUBLE NOT NULL), null:NULL)]) + LogicalFilter(condition=[IS NOT NULL($8)]) + LogicalSort(fetch=[5]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[>($t1, $t4)], expr#6=[CAST($t3):DOUBLE NOT NULL], expr#7=[/($t2, $t6)], expr#8=[null:NULL], expr#9=[CASE($t5, $t7, $t8)], ageTrend=[$t9]) + EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), $SUM0($0), COUNT($0)])]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NOT NULL($t8)], age=[$t8], $condition=[$t17]) + EnumerableLimit(fetch=[5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_sort_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_sort_push.json deleted file mode 100644 index babe36fd41b..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_sort_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(ageTrend=[CASE(>(COUNT() OVER (ROWS 1 PRECEDING), 1), /(SUM($8) OVER (ROWS 1 PRECEDING), CAST(COUNT($8) OVER (ROWS 1 PRECEDING)):DOUBLE NOT NULL), null:NULL)])\n LogicalFilter(condition=[IS NOT NULL($8)])\n LogicalSort(sort0=[$8], dir0=[ASC])\n LogicalSort(fetch=[5])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[>($t1, $t4)], expr#6=[CAST($t3):DOUBLE NOT NULL], expr#7=[/($t2, $t6)], expr#8=[null:NULL], expr#9=[CASE($t5, $t7, $t8)], ageTrend=[$t9])\n EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), $SUM0($0), COUNT($0)])])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NOT NULL($t8)], age=[$t8], $condition=[$t17])\n EnumerableSort(sort0=[$8], dir0=[ASC])\n EnumerableLimit(fetch=[5])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_sort_push.yaml new file mode 100644 index 00000000000..ea522249b3a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_trendline_sort_push.yaml @@ -0,0 +1,16 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(ageTrend=[CASE(>(COUNT() OVER (ROWS 1 PRECEDING), 1), /(SUM($8) OVER (ROWS 1 PRECEDING), CAST(COUNT($8) OVER (ROWS 1 PRECEDING)):DOUBLE NOT NULL), null:NULL)]) + LogicalFilter(condition=[IS NOT NULL($8)]) + LogicalSort(sort0=[$8], dir0=[ASC]) + LogicalSort(fetch=[5]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], expr#5=[>($t1, $t4)], expr#6=[CAST($t3):DOUBLE NOT NULL], expr#7=[/($t2, $t6)], expr#8=[null:NULL], expr#9=[CASE($t5, $t7, $t8)], ageTrend=[$t9]) + EnumerableWindow(window#0=[window(rows between $1 PRECEDING and CURRENT ROW aggs [COUNT(), $SUM0($0), COUNT($0)])]) + EnumerableCalc(expr#0..16=[{inputs}], expr#17=[IS NOT NULL($t8)], age=[$t8], $condition=[$t17]) + EnumerableSort(sort0=[$8], dir0=[ASC]) + EnumerableLimit(fetch=[5]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.json deleted file mode 100644 index 8d20573e03d..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[avg_age, state, city]" - }, - "children": [{ - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - }] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.yaml new file mode 100644 index 00000000000..b83985eeff3 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.yaml @@ -0,0 +1,18 @@ +root: + name: ProjectOperator + description: + fields: "[avg_age, state, city]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"\ + range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\"\ + :true,\"boost\":1.0}}},\"sort\":[],\"aggregations\":{\"composite_buckets\"\ + :{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\"\ + :\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"\ + order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\"\ + :true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\"\ + :{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.json deleted file mode 100644 index 8683e74106f..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[age]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"bool\":{\"filter\":[{\"range\":{\"balance\":{\"from\":10000,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":null,\"to\":40,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, needClean=true, searchDone=false, pitId=s9y3QQEhb3BlbnNlYXJjaC1zcWxfdGVzdF9pbmRleF9hY2NvdW50FkxkUmtkc296Ul9hY0ZyQWdnSklXTlEAFnJ6SVBMUjhrU3lTMHNMQXA1ckRnVWcAAAAAAAAAAAgWNEd5alM2OFhUZXVPYW1mSm1Gc1ZVUQEWTGRSa2Rzb3pSX2FjRnJBZ2dKSVdOUQAA, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.yaml new file mode 100644 index 00000000000..081f5398789 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[age]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\"\ + :{\"bool\":{\"filter\":[{\"bool\":{\"filter\":[{\"range\":{\"balance\":{\"\ + from\":10000,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"\ + boost\":1.0}}},{\"range\":{\"age\":{\"from\":null,\"to\":40,\"include_lower\"\ + :true,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\"\ + :false,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"\ + sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, needClean=true, searchDone=false,\ + \ pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_date_string.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_date_string.json deleted file mode 100644 index 71c47a5dce4..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_date_string.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "root" : { - "name" : "ProjectOperator", - "description" : { - "fields" : "[yyyy-MM-dd]" - }, - "children" : [ { - "name" : "FilterOperator", - "description" : { - "conditions" : "and(<(yyyy-MM-dd, cast_to_date(\"2018-11-09 00:00:00.000000000\")), >(yyyy-MM-dd, cast_to_date(\"2016-12-08 00:00:00.123456789\")))" - }, - "children" : [ { - "name" : "ProjectOperator", - "description" : { - "fields" : "[yyyy-MM-dd]" - }, - "children" : [ { - "name" : "OpenSearchIndexScan", - "description" : { - "request" : "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_date_formats, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"yyyy-MM-dd\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=s9y3QQEmb3BlbnNlYXJjaC1zcWxfdGVzdF9pbmRleF9kYXRlX2Zvcm1hdHMWNXlYSjJyR1VScENzVW9JcWpHMS12ZwAWak9VTVBiNnNRemFNR015bERWWWdRUQAAAAAAAAAAARZtQUN3TWs2d1FXbUZsbnNxYl9KbHNnARY1eVhKMnJHVVJwQ3NVb0lxakcxLXZnAAA=, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children" : [ ] - } ] - } ] - } ] - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_date_string.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_date_string.yaml new file mode 100644 index 00000000000..289c6297e6f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_date_string.yaml @@ -0,0 +1,22 @@ +root: + name: ProjectOperator + description: + fields: "[yyyy-MM-dd]" + children: + - name: FilterOperator + description: + conditions: "and(<(yyyy-MM-dd, cast_to_date(\"2018-11-09 00:00:00.000000000\"\ + )), >(yyyy-MM-dd, cast_to_date(\"2016-12-08 00:00:00.123456789\")))" + children: + - name: ProjectOperator + description: + fields: "[yyyy-MM-dd]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_date_formats,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"\ + _source\":{\"includes\":[\"yyyy-MM-dd\"],\"excludes\":[]}}, needClean=true,\ + \ searchDone=false, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_time_string.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_time_string.json deleted file mode 100644 index b54910e7321..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_time_string.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "root" : { - "name" : "ProjectOperator", - "description" : { - "fields" : "[custom_time]" - }, - "children" : [ { - "name" : "FilterOperator", - "description" : { - "conditions" : "and(<(custom_time, cast_to_time(\"2018-11-09 19:00:00.123456789\")), >(custom_time, cast_to_time(\"2016-12-08 12:00:00.123456789\")))" - }, - "children" : [ { - "name" : "ProjectOperator", - "description" : { - "fields" : "[custom_time]" - }, - "children" : [ { - "name" : "OpenSearchIndexScan", - "description" : { - "request" : "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_date_formats, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"custom_time\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=s9y3QQEmb3BlbnNlYXJjaC1zcWxfdGVzdF9pbmRleF9kYXRlX2Zvcm1hdHMWM2VGc1RiM2VRV0NYeFJyRGpMTHI1QQAWdXpkcXIzSWZSbU9xZDdrNzk2b3JkdwAAAAAAAAAAARZJUno2SEQ1RVN6Uy1JeGN4RE9HeUtBARYzZUZzVGIzZVFXQ1h4UnJEakxMcjVBAAA=, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children" : [ ] - } ] - } ] - } ] - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_time_string.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_time_string.yaml new file mode 100644 index 00000000000..c4b8521567a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_time_string.yaml @@ -0,0 +1,22 @@ +root: + name: ProjectOperator + description: + fields: "[custom_time]" + children: + - name: FilterOperator + description: + conditions: "and(<(custom_time, cast_to_time(\"2018-11-09 19:00:00.123456789\"\ + )), >(custom_time, cast_to_time(\"2016-12-08 12:00:00.123456789\")))" + children: + - name: ProjectOperator + description: + fields: "[custom_time]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_date_formats,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"\ + _source\":{\"includes\":[\"custom_time\"],\"excludes\":[]}}, needClean=true,\ + \ searchDone=false, pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_timestamp_string.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_timestamp_string.json deleted file mode 100644 index 795623cc05a..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_timestamp_string.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "root" : { - "name" : "ProjectOperator", - "description" : { - "fields" : "[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male]" - }, - "children" : [ { - "name" : "OpenSearchIndexScan", - "description" : { - "request" : "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_bank, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\":{\"birthdate\":{\"from\":null,\"to\":1541721600000,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},{\"range\":{\"birthdate\":{\"from\":1481155200000,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"birthdate\",\"gender\",\"city\",\"lastname\",\"balance\",\"employer\",\"state\",\"age\",\"email\",\"male\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, needClean=true, searchDone=false, pitId=s9y3QQEeb3BlbnNlYXJjaC1zcWxfdGVzdF9pbmRleF9iYW5rFmtHVFBVZFdaVEVTdzh4ZFVDYlFiU2cAFk8tSmZBa2hpUXRHOEFVTGdnMmxYUncAAAAAAAAAAAEWOFZwckxjWGhUaXVYRVBRWXptNFlQdwEWa0dUUFVkV1pURVN3OHhkVUNiUWJTZwAA, cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" - }, - "children" : [ ] - } ] - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_timestamp_string.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_timestamp_string.yaml new file mode 100644 index 00000000000..8c9655b232a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_timestamp_string.yaml @@ -0,0 +1,21 @@ +root: + name: ProjectOperator + description: + fields: "[account_number, firstname, address, birthdate, gender, city, lastname,\ + \ balance, employer, state, age, email, male]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_bank,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\"\ + :{\"bool\":{\"filter\":[{\"range\":{\"birthdate\":{\"from\":null,\"to\"\ + :1541721600000,\"include_lower\":true,\"include_upper\":false,\"boost\"\ + :1.0}}},{\"range\":{\"birthdate\":{\"from\":1481155200000,\"to\":null,\"\ + include_lower\":false,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\"\ + :true,\"boost\":1.0}},\"_source\":{\"includes\":[\"account_number\",\"firstname\"\ + ,\"address\",\"birthdate\",\"gender\",\"city\",\"lastname\",\"balance\"\ + ,\"employer\",\"state\",\"age\",\"email\",\"male\"],\"excludes\":[]},\"\ + sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, needClean=true, searchDone=false,\ + \ pitId=*,\ + \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_then_limit_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_then_limit_push.json deleted file mode 100644 index 6e46505faa6..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_then_limit_push.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[age]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, needClean=true, searchDone=false, pitId=null, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_then_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_then_limit_push.yaml new file mode 100644 index 00000000000..54ae6bccd3d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_then_limit_push.yaml @@ -0,0 +1,14 @@ +root: + name: ProjectOperator + description: + fields: "[age]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"query\":{\"\ + range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\"\ + :true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\"\ + :[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10_5_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10_5_push.json deleted file mode 100644 index 221687ef2ee..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10_5_push.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[age]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=null, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10_5_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10_5_push.yaml new file mode 100644 index 00000000000..604ec122fdf --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10_5_push.yaml @@ -0,0 +1,12 @@ +root: + name: ProjectOperator + description: + fields: "[age]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"\ + includes\":[\"age\"],\"excludes\":[]}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10_filter_5_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10_filter_5_push.json deleted file mode 100644 index b35143e5696..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10_filter_5_push.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[age]" - }, - "children": [ - { - "name": "LimitOperator", - "description": { - "limit": 5, - "offset": 0 - }, - "children": [ - { - "name": "FilterOperator", - "description": { - "conditions": ">(age, 30)" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10,\"timeout\":\"1m\"}, needClean=true, searchDone=false, pitId=null, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } - ] - } - ] - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10_filter_5_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10_filter_5_push.yaml new file mode 100644 index 00000000000..d1ba3213130 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10_filter_5_push.yaml @@ -0,0 +1,21 @@ +root: + name: ProjectOperator + description: + fields: "[age]" + children: + - name: LimitOperator + description: + limit: 5 + offset: 0 + children: + - name: FilterOperator + description: + conditions: ">(age, 30)" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":10,\"timeout\":\"1m\"}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10from1_10from2_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10from1_10from2_push.json deleted file mode 100644 index 330c72010d8..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10from1_10from2_push.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[age]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":3,\"size\":8,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=null, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10from1_10from2_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10from1_10from2_push.yaml new file mode 100644 index 00000000000..c6fbc4fd672 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10from1_10from2_push.yaml @@ -0,0 +1,12 @@ +root: + name: ProjectOperator + description: + fields: "[age]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":3,\"size\":8,\"timeout\":\"1m\",\"_source\":{\"\ + includes\":[\"age\"],\"excludes\":[]}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_5_10_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_5_10_push.json deleted file mode 100644 index 221687ef2ee..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_5_10_push.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[age]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=null, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_5_10_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_5_10_push.yaml new file mode 100644 index 00000000000..604ec122fdf --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_5_10_push.yaml @@ -0,0 +1,12 @@ +root: + name: ProjectOperator + description: + fields: "[age]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"\ + includes\":[\"age\"],\"excludes\":[]}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_offsets_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_offsets_push.json deleted file mode 100644 index b6675cc2e08..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_offsets_push.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[age]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":3,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=null, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_offsets_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_offsets_push.yaml new file mode 100644 index 00000000000..17960c2d39b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_offsets_push.yaml @@ -0,0 +1,12 @@ +root: + name: ProjectOperator + description: + fields: "[age]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":3,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"\ + includes\":[\"age\"],\"excludes\":[]}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.json deleted file mode 100644 index 02d26668776..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[ageMinus]" - }, - "children": [ - { - "name": "OpenSearchEvalOperator", - "description": { - "expressions": { - "ageMinus": "-(age, 30)" - } - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\"}, needClean\u003dtrue, searchDone\u003dfalse, pitId=null, cursorKeepAlive\u003dnull, searchAfter\u003dnull, searchResponse\u003dnull)" - }, - "children": [] - } - ] - } - ] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.yaml new file mode 100644 index 00000000000..2d14d36d922 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.yaml @@ -0,0 +1,17 @@ +root: + name: ProjectOperator + description: + fields: "[ageMinus]" + children: + - name: OpenSearchEvalOperator + description: + expressions: + ageMinus: "-(age, 30)" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\"}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_then_filter_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_then_filter_push.json deleted file mode 100644 index 989a2fe58b9..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_then_filter_push.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[age]" - }, - "children": [ - { - "name": "FilterOperator", - "description": { - "conditions": ">(age, 30)" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\"}, needClean=true, searchDone=false, pitId=null, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } - ] - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_then_filter_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_then_filter_push.yaml new file mode 100644 index 00000000000..f0f312cc4cd --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_then_filter_push.yaml @@ -0,0 +1,16 @@ +root: + name: ProjectOperator + description: + fields: "[age]" + children: + - name: FilterOperator + description: + conditions: ">(age, 30)" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\"}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_then_sort_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_then_sort_push.json deleted file mode 100644 index 802c41b923a..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_then_sort_push.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[age]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=null, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_then_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_then_sort_push.yaml new file mode 100644 index 00000000000..a49d48e5aa4 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_then_sort_push.yaml @@ -0,0 +1,13 @@ +root: + name: ProjectOperator + description: + fields: "[age]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"\ + includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\"\ + ,\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_output.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_output.json deleted file mode 100644 index a3d91813787..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_output.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[age2]" - }, - "children": [{ - "name": "DedupeOperator", - "description": { - "dedupeList": "[age2]", - "allowedDuplication": 1, - "keepEmpty": false, - "consecutive": false - }, - "children": [{ - "name": "OpenSearchEvalOperator", - "description": { - "expressions": { - "age2": "+(avg_age, 2)" - } - }, - "children": [{ - "name": "RemoveOperator", - "description": { - "removeList": "[city]" - }, - "children": [{ - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - }] - }] - }] - }] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_output.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_output.yaml new file mode 100644 index 00000000000..44b8bbf69a1 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_output.yaml @@ -0,0 +1,36 @@ +root: + name: ProjectOperator + description: + fields: "[age2]" + children: + - name: DedupeOperator + description: + dedupeList: "[age2]" + allowedDuplication: 1 + keepEmpty: false + consecutive: false + children: + - name: OpenSearchEvalOperator + description: + expressions: + age2: "+(avg_age, 2)" + children: + - name: RemoveOperator + description: + removeList: "[city]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"\ + query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\"\ + :false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[],\"\ + aggregations\":{\"composite_buckets\":{\"composite\":{\"size\"\ + :1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\"\ + ,\"missing_bucket\":true,\"missing_order\":\"first\",\"order\"\ + :\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\"\ + ,\"missing_bucket\":true,\"missing_order\":\"first\",\"order\"\ + :\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\"\ + :\"age\"}}}}}}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.json index 71a962c388d..e69de29bb2d 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.json @@ -1,15 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[pattern_count, sample_logs, patterns_field]" - }, - "children": [{ - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"patterns_field\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADZvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5wYXJzZS5QYXR0ZXJuc0V4cHJlc3Npb26h4+bazqpHBgIAAloAEHVzZUN1c3RvbVBhdHRlcm5MAAdwYXR0ZXJudAAZTGphdmEvdXRpbC9yZWdleC9QYXR0ZXJuO3hyADNvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5wYXJzZS5QYXJzZUV4cHJlc3Npb25977zy2Qz+ogIABEwACmlkZW50aWZpZXJ0ACpMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vRXhwcmVzc2lvbjtMAA1pZGVudGlmaWVyU3RydAASTGphdmEvbGFuZy9TdHJpbmc7TAAHcGF0dGVybnEAfgADTAALc291cmNlRmllbGRxAH4AA3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3QAEExqYXZhL3V0aWwvTGlzdDtMAAxmdW5jdGlvbk5hbWV0ADVMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25OYW1lO3hwc3IAPXNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZUxpc3QkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAVsACGVsZW1lbnRzdAATW0xqYXZhL2xhbmcvT2JqZWN0O3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAA3NyADFvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5SZWZlcmVuY2VFeHByZXNzaW9uq0TvXBIHhdYCAARMAARhdHRycQB+AARMAAVwYXRoc3EAfgAGTAAHcmF3UGF0aHEAfgAETAAEdHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hwdAAFZW1haWxzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWFxAH4ACnhwdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXEAfgARcQB+ABFzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AF3hwfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AB10AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAkeHB1cQB+AAwAAAAAdXEAfgAMAAAAAHNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AGH5xAH4AHHQABlNUUklOR35xAH4AIHQAB0tleXdvcmRxAH4AJXhzcgAvb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uTGl0ZXJhbEV4cHJlc3Npb25FQi3wjMeCJAIAAUwACWV4cHJWYWx1ZXQAKUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7eHBzcgAtb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuRXhwclN0cmluZ1ZhbHVlAEEyJXOJDhMCAAFMAAV2YWx1ZXEAfgAEeHIAL29yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkFic3RyYWN0RXhwclZhbHVlyWu1dgYURIoCAAB4cHQAAHNxAH4AMHNxAH4AM3QADnBhdHRlcm5zX2ZpZWxkc3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAR4cHQACHBhdHRlcm5zcQB+ADdxAH4AOXEAfgAycQB+ABAAcA==\\\"}\",\"lang\":\"opensearch_compounded_script\"},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"pattern_count\":{\"value_count\":{\"field\":\"_index\"}},\"sample_logs\":{\"top_hits\":{\"from\":0,\"size\":10,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"email\"],\"excludes\":[]}}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - }] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.yaml new file mode 100644 index 00000000000..7a2395fca4b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[pattern_count, sample_logs, patterns_field]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\"\ + :{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"patterns_field\"\ + :{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"\ + script\\\":\\\"rO0ABXNyADZvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5wYXJzZS5QYXR0ZXJuc0V4cHJlc3Npb26h4+bazqpHBgIAAloAEHVzZUN1c3RvbVBhdHRlcm5MAAdwYXR0ZXJudAAZTGphdmEvdXRpbC9yZWdleC9QYXR0ZXJuO3hyADNvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5wYXJzZS5QYXJzZUV4cHJlc3Npb25977zy2Qz+ogIABEwACmlkZW50aWZpZXJ0ACpMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vRXhwcmVzc2lvbjtMAA1pZGVudGlmaWVyU3RydAASTGphdmEvbGFuZy9TdHJpbmc7TAAHcGF0dGVybnEAfgADTAALc291cmNlRmllbGRxAH4AA3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3QAEExqYXZhL3V0aWwvTGlzdDtMAAxmdW5jdGlvbk5hbWV0ADVMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25OYW1lO3hwc3IAPXNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZUxpc3QkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAVsACGVsZW1lbnRzdAATW0xqYXZhL2xhbmcvT2JqZWN0O3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAA3NyADFvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5SZWZlcmVuY2VFeHByZXNzaW9uq0TvXBIHhdYCAARMAARhdHRycQB+AARMAAVwYXRoc3EAfgAGTAAHcmF3UGF0aHEAfgAETAAEdHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hwdAAFZW1haWxzcgAaamF2YS51dGlsLkFycmF5cyRBcnJheUxpc3TZpDy+zYgG0gIAAVsAAWFxAH4ACnhwdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXEAfgARcQB+ABFzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AF3hwfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AB10AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAkeHB1cQB+AAwAAAAAdXEAfgAMAAAAAHNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AGH5xAH4AHHQABlNUUklOR35xAH4AIHQAB0tleXdvcmRxAH4AJXhzcgAvb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uTGl0ZXJhbEV4cHJlc3Npb25FQi3wjMeCJAIAAUwACWV4cHJWYWx1ZXQAKUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7eHBzcgAtb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuRXhwclN0cmluZ1ZhbHVlAEEyJXOJDhMCAAFMAAV2YWx1ZXEAfgAEeHIAL29yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkFic3RyYWN0RXhwclZhbHVlyWu1dgYURIoCAAB4cHQAAHNxAH4AMHNxAH4AM3QADnBhdHRlcm5zX2ZpZWxkc3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAR4cHQACHBhdHRlcm5zcQB+ADdxAH4AOXEAfgAycQB+ABAAcA==\\\ + \"}\",\"lang\":\"opensearch_compounded_script\"},\"missing_bucket\":true,\"\ + missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"pattern_count\"\ + :{\"value_count\":{\"field\":\"_index\"}},\"sample_logs\":{\"top_hits\"\ + :{\"from\":0,\"size\":10,\"version\":false,\"seq_no_primary_term\":false,\"\ + explain\":false,\"_source\":{\"includes\":[\"email\"],\"excludes\":[]}}}}}}},\ + \ needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_count_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_count_push.json deleted file mode 100644 index 48f55af722e..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_count_push.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[age]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=null, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_count_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_count_push.yaml new file mode 100644 index 00000000000..a49d48e5aa4 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_count_push.yaml @@ -0,0 +1,13 @@ +root: + name: ProjectOperator + description: + fields: "[age]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"\ + includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\"\ + ,\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_then_limit_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_then_limit_push.json deleted file mode 100644 index 802c41b923a..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_then_limit_push.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[age]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=null, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_then_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_then_limit_push.yaml new file mode 100644 index 00000000000..a49d48e5aa4 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_then_limit_push.yaml @@ -0,0 +1,13 @@ +root: + name: ProjectOperator + description: + fields: "[age]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"\ + includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\"\ + ,\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_push.json deleted file mode 100644 index 2838d9c69b2..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_push.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[ageTrend]" - }, - "children": [ - { - "name": "TrendlineOperator", - "description": { - "computations": [ - { - "computationType" : "sma", - "numberOfDataPoints" : "2", - "dataField" : "age", - "alias" : "ageTrend" - } - ] - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\"}, needClean\u003dtrue, searchDone\u003dfalse, pitId=null, cursorKeepAlive\u003dnull, searchAfter\u003dnull, searchResponse\u003dnull)" - }, - "children": [] - } - ] - } - ] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_push.yaml new file mode 100644 index 00000000000..6a4ccc5b633 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_push.yaml @@ -0,0 +1,20 @@ +root: + name: ProjectOperator + description: + fields: "[ageTrend]" + children: + - name: TrendlineOperator + description: + computations: + - computationType: sma + numberOfDataPoints: "2" + dataField: age + alias: ageTrend + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\"}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_sort_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_sort_push.json deleted file mode 100644 index 4670dc25449..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_sort_push.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[ageTrend]" - }, - "children": [ - { - "name": "TrendlineOperator", - "description": { - "computations": [ - { - "computationType": "sma", - "numberOfDataPoints": "2", - "dataField": "age", - "alias": "ageTrend" - } - ] - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=null, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } - ] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_sort_push.yaml new file mode 100644 index 00000000000..47366bfd683 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_sort_push.yaml @@ -0,0 +1,21 @@ +root: + name: ProjectOperator + description: + fields: "[ageTrend]" + children: + - name: TrendlineOperator + description: + computations: + - computationType: sma + numberOfDataPoints: "2" + dataField: age + alias: ageTrend + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ + \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"sort\":[{\"\ + age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index 26eb187531d..5f9cdf8a5d9 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -58,7 +58,6 @@ import java.util.Set; import java.util.function.Supplier; import lombok.Getter; -import lombok.Setter; import org.apache.calcite.DataContext.Variable; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.rel.RelNode; @@ -915,7 +914,13 @@ private static boolean isColumn( interface Expression {} /** Main expression operators (like {@code equals}, {@code gt}, {@code exists} etc.) */ + @Getter public abstract static class QueryExpression implements Expression { + private int scriptCount; + + protected void accumulateScriptCount(int count) { + scriptCount += count; + } public abstract QueryBuilder builder(); @@ -1058,12 +1063,6 @@ static QueryExpression create(TerminalExpression expression) { throw new PredicateAnalyzerException(message); } } - - public static boolean containsScript(QueryExpression expression) { - return expression instanceof ScriptQueryExpression - || (expression instanceof CompoundQueryExpression - && ((CompoundQueryExpression) expression).containsScript()); - } } @Getter @@ -1103,15 +1102,12 @@ public static class CompoundQueryExpression extends QueryExpression { private final BoolQueryBuilder builder; @Getter private List analyzedNodes = new ArrayList<>(); @Getter private final List unAnalyzableNodes = new ArrayList<>(); - @Setter private boolean containsScript; public static CompoundQueryExpression or(QueryExpression... expressions) { CompoundQueryExpression bqe = new CompoundQueryExpression(false); for (QueryExpression expression : expressions) { bqe.builder.should(expression.builder()); - if (QueryExpression.containsScript(expression)) { - bqe.setContainsScript(true); - } + bqe.accumulateScriptCount(expression.getScriptCount()); } return bqe; } @@ -1130,27 +1126,19 @@ public static CompoundQueryExpression and(boolean partial, QueryExpression... ex bqe.unAnalyzableNodes.addAll(expression.getUnAnalyzableNodes()); if (!(expression instanceof UnAnalyzableQueryExpression)) { bqe.builder.must(expression.builder()); - if (QueryExpression.containsScript(expression)) { - bqe.setContainsScript(true); - } + bqe.accumulateScriptCount(expression.getScriptCount()); } } return bqe; } private CompoundQueryExpression(boolean partial) { - this(partial, boolQuery(), false); + this(partial, boolQuery()); } private CompoundQueryExpression(boolean partial, BoolQueryBuilder builder) { - this(partial, builder, false); - } - - private CompoundQueryExpression( - boolean partial, BoolQueryBuilder builder, boolean containsScript) { this.partial = partial; this.builder = requireNonNull(builder, "builder"); - this.containsScript = containsScript; } @Override @@ -1158,10 +1146,6 @@ public boolean isPartial() { return partial; } - public boolean containsScript() { - return containsScript; - } - @Override public QueryBuilder builder() { return builder; @@ -1474,6 +1458,7 @@ public ScriptQueryExpression( || rexNode.getKind().equals(SqlKind.IS_NOT_NULL))) { checkForNestedFieldOperands((RexCall) rexNode); } + accumulateScriptCount(1); RelJsonSerializer serializer = new RelJsonSerializer(cluster); this.codeGenerator = () -> diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java index 5c44b661f5e..d935ea7029e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java @@ -8,17 +8,15 @@ import static java.util.Objects.requireNonNull; import static org.opensearch.sql.common.setting.Settings.Key.CALCITE_PUSHDOWN_ROWCOUNT_ESTIMATION_FACTOR; -import java.util.ArrayDeque; import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.stream.IntStream; import java.util.stream.Stream; import lombok.Getter; import org.apache.calcite.adapter.enumerable.EnumerableMergeJoin; import org.apache.calcite.adapter.enumerable.EnumerableSort; import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptCost; +import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelCollations; @@ -35,31 +33,17 @@ import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.NumberUtil; -import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.search.aggregations.AggregationBuilder; -import org.opensearch.search.aggregations.AggregationBuilders; -import org.opensearch.search.aggregations.AggregatorFactories.Builder; -import org.opensearch.search.aggregations.BucketOrder; -import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; -import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; -import org.opensearch.search.aggregations.bucket.missing.MissingOrder; -import org.opensearch.search.aggregations.bucket.terms.MultiTermsAggregationBuilder; -import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; +import org.checkerframework.checker.nullness.qual.Nullable; import org.opensearch.search.sort.ScoreSortBuilder; import org.opensearch.search.sort.SortBuilder; import org.opensearch.search.sort.SortBuilders; import org.opensearch.search.sort.SortOrder; import org.opensearch.sql.common.setting.Settings.Key; import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; -import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder; -import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; import org.opensearch.sql.opensearch.storage.OpenSearchIndex; /** An abstract relational operator representing a scan of an OpenSearchIndex type. */ @@ -98,9 +82,7 @@ public RelDataType deriveRowType() { @Override public RelWriter explainTerms(RelWriter pw) { - OpenSearchRequestBuilder requestBuilder = osIndex.createRequestBuilder(); - pushDownContext.forEach(action -> action.apply(requestBuilder)); - String explainString = pushDownContext + ", " + requestBuilder; + String explainString = pushDownContext + ", " + pushDownContext.getRequestBuilder(); return super.explainTerms(pw) .itemIf("PushDownContext", explainString, !pushDownContext.isEmpty()); } @@ -109,8 +91,82 @@ protected Integer getQuerySizeLimit() { return osIndex.getSettings().getSettingValue(Key.QUERY_SIZE_LIMIT); } + /** + * Compute the final row count of the scan operator with the given push down operations. + * + *

The calculation logic tries to follow the same logic in Calcite. + */ @Override public double estimateRowCount(RelMetadataQuery mq) { + return pushDownContext.stream() + .reduce( + osIndex.getMaxResultWindow().doubleValue(), + (rowCount, operation) -> + switch (operation.type()) { + case AGGREGATION -> mq.getRowCount((RelNode) operation.digest()); + case PROJECT, SORT -> rowCount; + // Refer the org.apache.calcite.rel.metadata.RelMdRowCount + case COLLAPSE -> rowCount / 10; + case FILTER, SCRIPT -> NumberUtil.multiply( + rowCount, + RelMdUtil.guessSelectivity(((FilterDigest) operation.digest()).condition())); + case LIMIT -> Math.min(rowCount, ((LimitDigest) operation.digest()).limit()); + }, + (a, b) -> null); + } + + /** + * Compute the cost of the scan operator with the given push down operations. + * + *

We compute the final cost of the scan operator by accumulating the cost of each push down + * operation including aggregation, collapse, sort and script filter, and plus an external cost. + * The calculation logic tries to follow the same logic in Calcite. + * + *

While the left operations like project, filter, limit will be ignored in the accumulation + * process. But they will also affect the cost of cost-counted operations after them and the final + * external cost, which is calculated by `rows count * fields count`. + * + *

In the end, we still need to multiply the total cost by a factor to make the cost cheaper + * than non-pushdown operators. + */ + @Override + public @Nullable RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { + double dRows = osIndex.getMaxResultWindow().doubleValue(), dCpu = 0.0d; + for (PushDownOperation operation : pushDownContext) { + switch (operation.type()) { + case AGGREGATION -> { + dRows = mq.getRowCount((RelNode) operation.digest()); + dCpu += dRows * getAggMultiplier(operation); + } + // Ignored Project in cost accumulation, but it will affect the external cost + case PROJECT -> {} + case SORT -> dCpu += dRows; + // Refer the org.apache.calcite.rel.metadata.RelMdRowCount.getRowCount(Aggregate rel,...) + case COLLAPSE -> { + dRows = dRows / 10; + dCpu += dRows; + } + // Ignore cost the primitive filter but it will affect the rows count. + case FILTER -> dRows = + NumberUtil.multiply( + dRows, RelMdUtil.guessSelectivity(((FilterDigest) operation.digest()).condition())); + case SCRIPT -> { + FilterDigest filterDigest = (FilterDigest) operation.digest(); + dRows = NumberUtil.multiply(dRows, RelMdUtil.guessSelectivity(filterDigest.condition())); + // Calculate the cost of script filter by multiplying the selectivity of the filter and + // the factor amplified by script count. + dCpu += NumberUtil.multiply(dRows, Math.pow(1.1, filterDigest.scriptCount())); + } + // Ignore cost the LIMIT but it will affect the rows count. + // Try to reduce the rows count by 1 to make the cost cheaper slightly than non-push down. + // Because we'd like to push down LIMIT even when the fetch in LIMIT is greater than + // dRows. + case LIMIT -> dRows = Math.min(dRows, ((LimitDigest) operation.digest()).limit()) - 1; + } + ; + } + // Add the external cost to introduce the effect from FILTER, LIMIT and PROJECT. + dCpu += dRows * getRowType().getFieldList().size(); /* The impact factor to estimate the row count after push down an operator. @@ -120,31 +176,13 @@ public double estimateRowCount(RelMetadataQuery mq) { */ double estimateRowCountFactor = osIndex.getSettings().getSettingValue(CALCITE_PUSHDOWN_ROWCOUNT_ESTIMATION_FACTOR); - return pushDownContext.stream() - .reduce( - osIndex.getMaxResultWindow().doubleValue(), - (rowCount, action) -> - switch (action.type) { - case AGGREGATION -> mq.getRowCount((RelNode) action.digest) - * getAggMultiplier(action); - case PROJECT, SORT -> rowCount; - // Refer the org.apache.calcite.rel.core.Aggregate.estimateRowCount - case COLLAPSE -> rowCount * (1.0 - Math.pow(.5, 1)); - case FILTER -> NumberUtil.multiply( - rowCount, RelMdUtil.guessSelectivity((RexNode) action.digest)); - case SCRIPT -> NumberUtil.multiply( - rowCount, RelMdUtil.guessSelectivity((RexNode) action.digest)) - * 1.1; - case LIMIT -> Math.min(rowCount, ((LimitDigest) action.digest).limit()); - } - * estimateRowCountFactor, - (a, b) -> null); + return planner.getCostFactory().makeCost(dCpu * estimateRowCountFactor, 0, 0); } /** See source in {@link org.apache.calcite.rel.core.Aggregate::computeSelfCost} */ - private static float getAggMultiplier(PushDownAction action) { + private static float getAggMultiplier(PushDownOperation operation) { // START CALCITE - List aggCalls = ((Aggregate) action.digest).getAggCallList(); + List aggCalls = ((Aggregate) operation.digest()).getAggCallList(); float multiplier = 1f + (float) aggCalls.size() * 0.125f; for (AggregateCall aggCall : aggCalls) { if (aggCall.getAggregation().getName().equals("SUM")) { @@ -155,49 +193,12 @@ private static float getAggMultiplier(PushDownAction action) { } // END CALCITE - // For script aggregation, we need to multiply the multiplier by 2.2 to make up the cost. As we + // For script aggregation, we need to multiply the multiplier by 1.1 to make up the cost. As we // prefer to have non-script agg push down after optimized by {@link PPLAggregateConvertRule} - if (((AggPushDownAction) action.action).isScriptPushed) { - multiplier *= 2.2f; - } + multiplier *= (float) Math.pow(1.1f, ((AggPushDownAction) operation.action()).getScriptCount()); return multiplier; } - // TODO: should we consider equivalent among PushDownContexts with different push down sequence? - public static class PushDownContext extends ArrayDeque { - - @Getter private boolean isAggregatePushed = false; - @Getter private AggPushDownAction aggPushDownAction; - @Getter private boolean isLimitPushed = false; - @Getter private boolean isProjectPushed = false; - @Getter private int startFrom = 0; - - @Override - public PushDownContext clone() { - return (PushDownContext) super.clone(); - } - - @Override - public boolean add(PushDownAction pushDownAction) { - if (pushDownAction.type == PushDownType.AGGREGATION) { - isAggregatePushed = true; - this.aggPushDownAction = (AggPushDownAction) pushDownAction.action; - } - if (pushDownAction.type == PushDownType.LIMIT) { - isLimitPushed = true; - startFrom += ((LimitDigest) pushDownAction.digest).offset(); - } - if (pushDownAction.type == PushDownType.PROJECT) { - isProjectPushed = true; - } - return super.add(pushDownAction); - } - - public boolean containsDigest(Object digest) { - return this.stream().anyMatch(action -> action.digest.equals(digest)); - } - } - protected abstract AbstractCalciteIndexScan buildScan( RelOptCluster cluster, RelTraitSet traitSet, @@ -244,22 +245,6 @@ private static boolean isAnyCollationNameInAggregateOutput( .reduce(false, Boolean::logicalOr); } - /** - * Create a new {@link PushDownContext} without the collation action. - * - * @param pushDownContext The original push-down context. - * @return A new push-down context without the collation action. - */ - protected PushDownContext cloneWithoutSort(PushDownContext pushDownContext) { - PushDownContext newContext = new PushDownContext(); - for (PushDownAction action : pushDownContext) { - if (action.type() != PushDownType.SORT) { - newContext.add(action); - } - } - return newContext; - } - /** * The sort pushdown is not only applied in logical plan side, but also should be applied in * physical plan side. Because we could push down the {@link EnumerableSort} of {@link @@ -285,15 +270,16 @@ public AbstractCalciteIndexScan pushDownSort(List collations) osIndex, getRowType(), // Existing collations are overridden (discarded) by the new collations, - cloneWithoutSort(pushDownContext)); + pushDownContext.cloneWithoutSort()); - AbstractAction action; + AbstractAction action; Object digest; - if (pushDownContext.isAggregatePushed) { + if (pushDownContext.isAggregatePushed()) { // Push down the sort into the aggregation bucket - this.pushDownContext.aggPushDownAction.pushDownSortIntoAggBucket( - collations, getRowType().getFieldNames()); - action = requestBuilder -> {}; + action = + (AggregationBuilderAction) + aggAction -> + aggAction.pushDownSortIntoAggBucket(collations, getRowType().getFieldNames()); digest = collations; } else { List> builders = new ArrayList<>(); @@ -322,10 +308,10 @@ public AbstractCalciteIndexScan pushDownSort(List collations) } builders.add(sortBuilder.order(order)); } - action = requestBuilder -> requestBuilder.pushDownSort(builders); + action = (OSRequestBuilderAction) requestBuilder -> requestBuilder.pushDownSort(builders); digest = builders.toString(); } - newScan.pushDownContext.add(PushDownAction.of(PushDownType.SORT, digest, action)); + newScan.pushDownContext.add(PushDownType.SORT, digest, action); return newScan; } catch (Exception e) { if (LOG.isDebugEnabled()) { @@ -334,188 +320,4 @@ public AbstractCalciteIndexScan pushDownSort(List collations) } return null; } - - protected enum PushDownType { - FILTER, - PROJECT, - AGGREGATION, - SORT, - LIMIT, - SCRIPT, - COLLAPSE - // HIGHLIGHT, - // NESTED - } - - /** - * Represents a push down action that can be applied to an OpenSearchRequestBuilder. - * - * @param type PushDownType enum - * @param digest the digest of the pushed down operator - * @param action the lambda action to apply on the OpenSearchRequestBuilder - */ - public record PushDownAction(PushDownType type, Object digest, AbstractAction action) { - static PushDownAction of(PushDownType type, Object digest, AbstractAction action) { - return new PushDownAction(type, digest, action); - } - - public String toString() { - return type + "->" + digest; - } - - public void apply(OpenSearchRequestBuilder requestBuilder) { - action.apply(requestBuilder); - } - } - - public interface AbstractAction { - void apply(OpenSearchRequestBuilder requestBuilder); - } - - public record LimitDigest(int limit, int offset) { - @Override - public String toString() { - return offset == 0 ? String.valueOf(limit) : "[" + limit + " from " + offset + "]"; - } - } - - // TODO: shall we do deep copy for this action since it's mutable? - public static class AggPushDownAction implements AbstractAction { - - private Pair, OpenSearchAggregationResponseParser> aggregationBuilder; - private final Map extendedTypeMapping; - @Getter private final boolean isScriptPushed; - // Record the output field names of all buckets as the sequence of buckets - private List bucketNames; - - public AggPushDownAction( - Pair, OpenSearchAggregationResponseParser> aggregationBuilder, - Map extendedTypeMapping, - List bucketNames) { - this.aggregationBuilder = aggregationBuilder; - this.extendedTypeMapping = extendedTypeMapping; - this.isScriptPushed = - aggregationBuilder.getLeft().stream().anyMatch(this::isScriptAggBuilder); - this.bucketNames = bucketNames; - } - - private boolean isScriptAggBuilder(AggregationBuilder aggBuilder) { - return aggBuilder instanceof ValuesSourceAggregationBuilder valueSourceAgg - && valueSourceAgg.script() != null; - } - - @Override - public void apply(OpenSearchRequestBuilder requestBuilder) { - requestBuilder.pushDownAggregation(aggregationBuilder); - requestBuilder.pushTypeMapping(extendedTypeMapping); - } - - public void pushDownSortIntoAggBucket( - List collations, List fieldNames) { - // aggregationBuilder.getLeft() could be empty when count agg optimization works - if (aggregationBuilder.getLeft().isEmpty()) return; - AggregationBuilder builder = aggregationBuilder.getLeft().getFirst(); - List selected = new ArrayList<>(collations.size()); - if (builder instanceof CompositeAggregationBuilder compositeAggBuilder) { - // It will always use a single CompositeAggregationBuilder for the aggregation with GroupBy - // See {@link AggregateAnalyzer} - List> buckets = compositeAggBuilder.sources(); - List> newBuckets = new ArrayList<>(buckets.size()); - List newBucketNames = new ArrayList<>(buckets.size()); - // Have to put the collation required buckets first, then the rest of buckets. - collations.forEach( - collation -> { - /* - Must find the bucket by field name because: - 1. The sequence of buckets may have changed after sort push-down. - 2. The schema of scan operator may be inconsistent with the sequence of buckets - after project push-down. - */ - String bucketName = fieldNames.get(collation.getFieldIndex()); - CompositeValuesSourceBuilder bucket = buckets.get(bucketNames.indexOf(bucketName)); - Direction direction = collation.getDirection(); - NullDirection nullDirection = collation.nullDirection; - SortOrder order = - Direction.DESCENDING.equals(direction) ? SortOrder.DESC : SortOrder.ASC; - if (bucket.missingBucket()) { - MissingOrder missingOrder = - switch (nullDirection) { - case FIRST -> MissingOrder.FIRST; - case LAST -> MissingOrder.LAST; - default -> MissingOrder.DEFAULT; - }; - bucket.missingOrder(missingOrder); - } - newBuckets.add(bucket.order(order)); - newBucketNames.add(bucketName); - selected.add(bucketName); - }); - IntStream.range(0, buckets.size()) - .mapToObj(fieldNames::get) - .filter(name -> !selected.contains(name)) - .forEach( - name -> { - newBuckets.add(buckets.get(bucketNames.indexOf(name))); - newBucketNames.add(name); - }); - Builder newAggBuilder = new Builder(); - compositeAggBuilder.getSubAggregations().forEach(newAggBuilder::addAggregator); - aggregationBuilder = - Pair.of( - Collections.singletonList( - AggregationBuilders.composite("composite_buckets", newBuckets) - .subAggregations(newAggBuilder) - .size(compositeAggBuilder.size())), - aggregationBuilder.getRight()); - bucketNames = newBucketNames; - } - if (builder instanceof TermsAggregationBuilder termsAggBuilder) { - termsAggBuilder.order( - BucketOrder.key(!collations.getFirst().getDirection().isDescending())); - } - // TODO for MultiTermsAggregationBuilder - } - - /** - * Check if the limit can be pushed down into aggregation bucket when the limit size is less - * than bucket number. - */ - public boolean pushDownLimitIntoBucketSize(Integer size) { - // aggregationBuilder.getLeft() could be empty when count agg optimization works - if (aggregationBuilder.getLeft().isEmpty()) return false; - AggregationBuilder builder = aggregationBuilder.getLeft().getFirst(); - if (builder instanceof CompositeAggregationBuilder compositeAggBuilder) { - if (size < compositeAggBuilder.size()) { - compositeAggBuilder.size(size); - return true; - } else { - return false; - } - } - if (builder instanceof TermsAggregationBuilder termsAggBuilder) { - if (size < termsAggBuilder.size()) { - termsAggBuilder.size(size); - return true; - } else { - return false; - } - } - if (builder instanceof MultiTermsAggregationBuilder multiTermsAggBuilder) { - if (size < multiTermsAggBuilder.size()) { - multiTermsAggBuilder.size(size); - return true; - } else { - return false; - } - } - // now we only have Composite, Terms and MultiTerms bucket aggregations, - // add code here when we could support more in the future. - if (builder instanceof ValuesSourceAggregationBuilder.LeafOnly) { - // Note: all metric aggregations will be treated as pushed since it generates only one row. - return true; - } - throw new OpenSearchRequestBuilder.PushDownUnSupportedException( - "Unknown aggregation builder " + builder.getClass().getSimpleName()); - } - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java index 0fd326306dc..7fb32cc761a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java @@ -105,8 +105,7 @@ public Result implement(EnumerableRelImplementor implementor, Prefer pref) { return new AbstractEnumerable<>() { @Override public Enumerator enumerator() { - OpenSearchRequestBuilder requestBuilder = osIndex.createRequestBuilder(); - pushDownContext.forEach(action -> action.apply(requestBuilder)); + OpenSearchRequestBuilder requestBuilder = getOrCreateRequestBuilder(); return new OpenSearchIndexEnumerator( osIndex.getClient(), getFieldPath(), @@ -123,4 +122,28 @@ private List getFieldPath() { .map(f -> osIndex.getAliasMapping().getOrDefault(f, f)) .toList(); } + + /** + * In some edge cases where the digests of more than one scan are the same, and then the Calcite + * planner will reuse the same scan along with the same PushDownContext inner it. However, the + * `OpenSearchRequestBuilder` inner `PushDownContext` is not reusable since it has status changed + * in the search process. + * + *

To avoid this issue and try to construct `OpenSearchRequestBuilder` as less as possible, + * this method will get and reuse the `OpenSearchRequestBuilder` in PushDownContext for the first + * time, and then construct new ones for the following invoking. + * + * @return OpenSearchRequestBuilder to be used by enumerator + */ + private volatile boolean isRequestBuilderUsedByEnumerator = false; + + private OpenSearchRequestBuilder getOrCreateRequestBuilder() { + synchronized (this.pushDownContext) { + if (isRequestBuilderUsedByEnumerator) { + return this.pushDownContext.createRequestBuilder(); + } + isRequestBuilderUsedByEnumerator = true; + return this.pushDownContext.getRequestBuilder(); + } + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java index 109bf9d59b3..ee9f6be144c 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java @@ -49,14 +49,10 @@ import org.opensearch.sql.opensearch.planner.physical.EnumerableIndexScanRule; import org.opensearch.sql.opensearch.planner.physical.OpenSearchIndexRules; import org.opensearch.sql.opensearch.request.AggregateAnalyzer; -import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder; import org.opensearch.sql.opensearch.request.PredicateAnalyzer; import org.opensearch.sql.opensearch.request.PredicateAnalyzer.QueryExpression; import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; import org.opensearch.sql.opensearch.storage.OpenSearchIndex; -import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan.LimitDigest; -import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan.PushDownAction; -import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan.PushDownType; /** The logical relational operator representing a scan of an OpenSearchIndex type. */ @Getter @@ -72,7 +68,7 @@ public CalciteLogicalIndexScan( table, osIndex, table.getRowType(), - new PushDownContext()); + new PushDownContext(osIndex)); } protected CalciteLogicalIndexScan( @@ -135,15 +131,15 @@ public AbstractRelNode pushDownFilter(Filter filter) { filter.getCondition(), schema, fieldTypes, rowType, getCluster()); // TODO: handle the case where condition contains a score function newScan.pushDownContext.add( - PushDownAction.of( - QueryExpression.containsScript(queryExpression) - ? PushDownType.SCRIPT - : PushDownType.FILTER, + queryExpression.getScriptCount() > 0 ? PushDownType.SCRIPT : PushDownType.FILTER, + new FilterDigest( + queryExpression.getScriptCount(), queryExpression.isPartial() ? constructCondition( queryExpression.getAnalyzedNodes(), getCluster().getRexBuilder()) - : filter.getCondition(), - requestBuilder -> requestBuilder.pushDownFilter(queryExpression.builder()))); + : filter.getCondition()), + (OSRequestBuilderAction) + requestBuilder -> requestBuilder.pushDownFilter(queryExpression.builder())); // If the query expression is partial, we need to replace the input of the filter with the // partial pushed scan and the filter condition with non-pushed-down conditions. @@ -199,10 +195,9 @@ public CalciteLogicalIndexScan pushDownCollapse(Project finalOutput, String fiel } CalciteLogicalIndexScan newScan = this.copyWithNewSchema(finalOutput.getRowType()); newScan.pushDownContext.add( - PushDownAction.of( - PushDownType.COLLAPSE, - fieldName, - requestBuilder -> requestBuilder.pushDownCollapse(field))); + PushDownType.COLLAPSE, + fieldName, + (OSRequestBuilderAction) requestBuilder -> requestBuilder.pushDownCollapse(field)); return newScan; } @@ -243,7 +238,7 @@ public CalciteLogicalIndexScan pushDownProject(List selectedColumns) { AbstractAction action; if (pushDownContext.isAggregatePushed()) { // For aggregate, we do nothing on query builder but only change the schema of the scan. - action = requestBuilder -> {}; + action = (AggregationBuilderAction) aggAction -> {}; } else { Map aliasMapping = this.osIndex.getAliasMapping(); // For alias types, we need to push down its original path instead of the alias name. @@ -251,10 +246,11 @@ public CalciteLogicalIndexScan pushDownProject(List selectedColumns) { newSchema.getFieldNames().stream() .map(fieldName -> aliasMapping.getOrDefault(fieldName, fieldName)) .toList(); - action = requestBuilder -> requestBuilder.pushDownProjectStream(projectedFields.stream()); + action = + (OSRequestBuilderAction) + requestBuilder -> requestBuilder.pushDownProjectStream(projectedFields.stream()); } - newScan.pushDownContext.add( - PushDownAction.of(PushDownType.PROJECT, newSchema.getFieldNames(), action)); + newScan.pushDownContext.add(PushDownType.PROJECT, newSchema.getFieldNames(), action); return newScan; } @@ -287,7 +283,7 @@ public AbstractRelNode pushDownAggregate(Aggregate aggregate, Project project) { osIndex, aggregate.getRowType(), // Aggregation will eliminate all collations. - cloneWithoutSort(pushDownContext)); + pushDownContext.cloneWithoutSort()); Map fieldTypes = this.osIndex.getFieldTypes(); List outputFields = aggregate.getRowType().getFieldNames(); final Pair, OpenSearchAggregationResponseParser> aggregationBuilder = @@ -306,8 +302,8 @@ public AbstractRelNode pushDownAggregate(Aggregate aggregate, Project project) { new AggPushDownAction( aggregationBuilder, extendedTypeMapping, - outputFields.subList(0, aggregate.getGroupSet().length())); - newScan.pushDownContext.add(PushDownAction.of(PushDownType.AGGREGATION, aggregate, action)); + outputFields.subList(0, aggregate.getGroupSet().cardinality())); + newScan.pushDownContext.add(PushDownType.AGGREGATION, aggregate, action); if (aggregationBuilder.getLeft().size() == 1 && aggregationBuilder.getLeft().getFirst() instanceof AutoDateHistogramAggregationBuilder autoDateHistogram) { @@ -338,29 +334,27 @@ public AbstractRelNode pushDownAggregate(Aggregate aggregate, Project project) { public AbstractRelNode pushDownLimit(LogicalSort sort, Integer limit, Integer offset) { try { if (pushDownContext.isAggregatePushed()) { - // Push down the limit into the aggregation bucket - boolean pushed = + // Push down the limit into the aggregation bucket in advance to detect whether the limit + // can update the aggregation builder + boolean updated = pushDownContext.getAggPushDownAction().pushDownLimitIntoBucketSize(limit + offset); - if (!pushed && offset > 0) return null; + if (!updated && offset > 0) return null; CalciteLogicalIndexScan newScan = this.copyWithNewSchema(getRowType()); - newScan.pushDownContext.add( - PushDownAction.of( - PushDownType.LIMIT, new LimitDigest(limit, offset), requestBuilder -> {})); + // Simplify the action if it doesn't update the aggregation builder, otherwise keep the + // original action + // It won't change the aggregation builder by do this action again since it's idempotent + AggregationBuilderAction action = + updated + ? aggAction -> aggAction.pushDownLimitIntoBucketSize(limit + offset) + : aggAction -> {}; + newScan.pushDownContext.add(PushDownType.LIMIT, new LimitDigest(limit, offset), action); return offset > 0 ? sort.copy(sort.getTraitSet(), List.of(newScan)) : newScan; } else { CalciteLogicalIndexScan newScan = this.copyWithNewSchema(getRowType()); - int newStartFrom = newScan.pushDownContext.getStartFrom() + offset; - if (newStartFrom >= newScan.osIndex.getMaxResultWindow()) { - throw new OpenSearchRequestBuilder.PushDownUnSupportedException( - String.format( - "Requested offset %d should be less than the max result window %d", - newStartFrom, newScan.osIndex.getMaxResultWindow())); - } newScan.pushDownContext.add( - PushDownAction.of( - PushDownType.LIMIT, - new LimitDigest(limit, offset), - requestBuilder -> requestBuilder.pushDownLimit(limit, offset))); + PushDownType.LIMIT, + new LimitDigest(limit, offset), + (OSRequestBuilderAction) requestBuilder -> requestBuilder.pushDownLimit(limit, offset)); return newScan; } } catch (Exception e) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/PushDownContext.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/PushDownContext.java new file mode 100644 index 00000000000..9819f893e1e --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/PushDownContext.java @@ -0,0 +1,342 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.scan; + +import com.google.common.collect.Iterators; +import java.util.AbstractCollection; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.IntStream; +import lombok.Getter; +import org.apache.calcite.rel.RelFieldCollation; +import org.apache.calcite.rel.RelFieldCollation.Direction; +import org.apache.calcite.rel.RelFieldCollation.NullDirection; +import org.apache.calcite.rex.RexNode; +import org.apache.commons.lang3.tuple.Pair; +import org.jetbrains.annotations.NotNull; +import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.AggregationBuilders; +import org.opensearch.search.aggregations.AggregatorFactories.Builder; +import org.opensearch.search.aggregations.BucketOrder; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; +import org.opensearch.search.aggregations.bucket.missing.MissingOrder; +import org.opensearch.search.aggregations.bucket.terms.MultiTermsAggregationBuilder; +import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; +import org.opensearch.search.sort.SortOrder; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder; +import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; +import org.opensearch.sql.opensearch.storage.OpenSearchIndex; + +@Getter +public class PushDownContext extends AbstractCollection { + private final OpenSearchIndex osIndex; + private final OpenSearchRequestBuilder requestBuilder; + private ArrayDeque operationsForRequestBuilder; + + private boolean isAggregatePushed = false; + private AggPushDownAction aggPushDownAction; + private ArrayDeque operationsForAgg; + + private boolean isLimitPushed = false; + private boolean isProjectPushed = false; + + public PushDownContext(OpenSearchIndex osIndex) { + this.osIndex = osIndex; + this.requestBuilder = osIndex.createRequestBuilder(); + } + + @Override + public PushDownContext clone() { + PushDownContext newContext = new PushDownContext(osIndex); + newContext.addAll(this); + return newContext; + } + + /** + * Create a new {@link PushDownContext} without the collation action. + * + * @return A new push-down context without the collation action. + */ + public PushDownContext cloneWithoutSort() { + PushDownContext newContext = new PushDownContext(osIndex); + for (PushDownOperation action : this) { + if (action.type() != PushDownType.SORT) { + newContext.add(action); + } + } + return newContext; + } + + @NotNull + @Override + public Iterator iterator() { + if (operationsForRequestBuilder == null) { + return Collections.emptyIterator(); + } else if (operationsForAgg == null) { + return operationsForRequestBuilder.iterator(); + } else { + return Iterators.concat(operationsForRequestBuilder.iterator(), operationsForAgg.iterator()); + } + } + + @Override + public int size() { + return (operationsForRequestBuilder == null ? 0 : operationsForRequestBuilder.size()) + + (operationsForAgg == null ? 0 : operationsForAgg.size()); + } + + ArrayDeque getOperationsForRequestBuilder() { + if (operationsForRequestBuilder == null) { + this.operationsForRequestBuilder = new ArrayDeque<>(); + } + return operationsForRequestBuilder; + } + + ArrayDeque getOperationsForAgg() { + if (operationsForAgg == null) { + this.operationsForAgg = new ArrayDeque<>(); + } + return operationsForAgg; + } + + @Override + public boolean add(PushDownOperation operation) { + if (operation.type() == PushDownType.AGGREGATION) { + isAggregatePushed = true; + this.aggPushDownAction = (AggPushDownAction) operation.action(); + } + if (operation.type() == PushDownType.LIMIT) { + isLimitPushed = true; + } + if (operation.type() == PushDownType.PROJECT) { + isProjectPushed = true; + } + operation.action().transform(this, operation); + return true; + } + + void add(PushDownType type, Object digest, AbstractAction action) { + add(new PushDownOperation(type, digest, action)); + } + + public boolean containsDigest(Object digest) { + return this.stream().anyMatch(action -> action.digest().equals(digest)); + } + + public OpenSearchRequestBuilder createRequestBuilder() { + OpenSearchRequestBuilder newRequestBuilder = osIndex.createRequestBuilder(); + if (operationsForRequestBuilder != null) { + operationsForRequestBuilder.forEach( + operation -> ((OSRequestBuilderAction) operation.action()).apply(newRequestBuilder)); + } + return newRequestBuilder; + } +} + +enum PushDownType { + FILTER, + PROJECT, + AGGREGATION, + SORT, + LIMIT, + SCRIPT, + COLLAPSE + // HIGHLIGHT, + // NESTED +} + +/** + * Represents a push down operation that can be applied to an OpenSearchRequestBuilder. + * + * @param type PushDownType enum + * @param digest the digest of the pushed down operator + * @param action the lambda action to apply on the OpenSearchRequestBuilder + */ +record PushDownOperation(PushDownType type, Object digest, AbstractAction action) { + public String toString() { + return type + "->" + digest; + } +} + +interface AbstractAction { + void apply(T target); + + void transform(PushDownContext context, PushDownOperation operation); +} + +interface OSRequestBuilderAction extends AbstractAction { + default void transform(PushDownContext context, PushDownOperation operation) { + apply(context.getRequestBuilder()); + context.getOperationsForRequestBuilder().add(operation); + } +} + +interface AggregationBuilderAction extends AbstractAction { + default void transform(PushDownContext context, PushDownOperation operation) { + apply(context.getAggPushDownAction()); + context.getOperationsForAgg().add(operation); + } +} + +record FilterDigest(int scriptCount, RexNode condition) { + @Override + public String toString() { + return condition.toString(); + } +} + +record LimitDigest(int limit, int offset) { + @Override + public String toString() { + return offset == 0 ? String.valueOf(limit) : "[" + limit + " from " + offset + "]"; + } +} + +// TODO: shall we do deep copy for this action since it's mutable? +class AggPushDownAction implements OSRequestBuilderAction { + + private Pair, OpenSearchAggregationResponseParser> aggregationBuilder; + private final Map extendedTypeMapping; + @Getter private final long scriptCount; + // Record the output field names of all buckets as the sequence of buckets + private List bucketNames; + + public AggPushDownAction( + Pair, OpenSearchAggregationResponseParser> aggregationBuilder, + Map extendedTypeMapping, + List bucketNames) { + this.aggregationBuilder = aggregationBuilder; + this.extendedTypeMapping = extendedTypeMapping; + this.scriptCount = + aggregationBuilder.getLeft().stream().filter(this::isScriptAggBuilder).count(); + this.bucketNames = bucketNames; + } + + private boolean isScriptAggBuilder(AggregationBuilder aggBuilder) { + return aggBuilder instanceof ValuesSourceAggregationBuilder valueSourceAgg + && valueSourceAgg.script() != null; + } + + @Override + public void apply(OpenSearchRequestBuilder requestBuilder) { + requestBuilder.pushDownAggregation(aggregationBuilder); + requestBuilder.pushTypeMapping(extendedTypeMapping); + } + + public void pushDownSortIntoAggBucket( + List collations, List fieldNames) { + // aggregationBuilder.getLeft() could be empty when count agg optimization works + if (aggregationBuilder.getLeft().isEmpty()) return; + AggregationBuilder builder = aggregationBuilder.getLeft().getFirst(); + List selected = new ArrayList<>(collations.size()); + if (builder instanceof CompositeAggregationBuilder compositeAggBuilder) { + // It will always use a single CompositeAggregationBuilder for the aggregation with GroupBy + // See {@link AggregateAnalyzer} + List> buckets = compositeAggBuilder.sources(); + List> newBuckets = new ArrayList<>(buckets.size()); + List newBucketNames = new ArrayList<>(buckets.size()); + // Have to put the collation required buckets first, then the rest of buckets. + collations.forEach( + collation -> { + /* + Must find the bucket by field name because: + 1. The sequence of buckets may have changed after sort push-down. + 2. The schema of scan operator may be inconsistent with the sequence of buckets + after project push-down. + */ + String bucketName = fieldNames.get(collation.getFieldIndex()); + CompositeValuesSourceBuilder bucket = buckets.get(bucketNames.indexOf(bucketName)); + Direction direction = collation.getDirection(); + NullDirection nullDirection = collation.nullDirection; + SortOrder order = + Direction.DESCENDING.equals(direction) ? SortOrder.DESC : SortOrder.ASC; + if (bucket.missingBucket()) { + MissingOrder missingOrder = + switch (nullDirection) { + case FIRST -> MissingOrder.FIRST; + case LAST -> MissingOrder.LAST; + default -> MissingOrder.DEFAULT; + }; + bucket.missingOrder(missingOrder); + } + newBuckets.add(bucket.order(order)); + newBucketNames.add(bucketName); + selected.add(bucketName); + }); + IntStream.range(0, buckets.size()) + .mapToObj(fieldNames::get) + .filter(name -> !selected.contains(name)) + .forEach( + name -> { + newBuckets.add(buckets.get(bucketNames.indexOf(name))); + newBucketNames.add(name); + }); + Builder newAggBuilder = new Builder(); + compositeAggBuilder.getSubAggregations().forEach(newAggBuilder::addAggregator); + aggregationBuilder = + Pair.of( + Collections.singletonList( + AggregationBuilders.composite("composite_buckets", newBuckets) + .subAggregations(newAggBuilder) + .size(compositeAggBuilder.size())), + aggregationBuilder.getRight()); + bucketNames = newBucketNames; + } + if (builder instanceof TermsAggregationBuilder termsAggBuilder) { + termsAggBuilder.order(BucketOrder.key(!collations.getFirst().getDirection().isDescending())); + } + // TODO for MultiTermsAggregationBuilder + } + + /** + * Check if the limit can be pushed down into aggregation bucket when the limit size is less than + * bucket number. + */ + public boolean pushDownLimitIntoBucketSize(Integer size) { + // aggregationBuilder.getLeft() could be empty when count agg optimization works + if (aggregationBuilder.getLeft().isEmpty()) return false; + AggregationBuilder builder = aggregationBuilder.getLeft().getFirst(); + if (builder instanceof CompositeAggregationBuilder compositeAggBuilder) { + if (size < compositeAggBuilder.size()) { + compositeAggBuilder.size(size); + return true; + } else { + return false; + } + } + if (builder instanceof TermsAggregationBuilder termsAggBuilder) { + if (size < termsAggBuilder.size()) { + termsAggBuilder.size(size); + return true; + } else { + return false; + } + } + if (builder instanceof MultiTermsAggregationBuilder multiTermsAggBuilder) { + if (size < multiTermsAggBuilder.size()) { + multiTermsAggBuilder.size(size); + return true; + } else { + return false; + } + } + // now we only have Composite, Terms and MultiTerms bucket aggregations, + // add code here when we could support more in the future. + if (builder instanceof ValuesSourceAggregationBuilder.LeafOnly) { + // Note: all metric aggregations will be treated as pushed since it generates only one row. + return true; + } + throw new OpenSearchRequestBuilder.PushDownUnSupportedException( + "Unknown aggregation builder " + builder.getClass().getSimpleName()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/CalciteIndexScanCostTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/CalciteIndexScanCostTest.java new file mode 100644 index 00000000000..f02b40b9eae --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/CalciteIndexScanCostTest.java @@ -0,0 +1,547 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.scan; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyDouble; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableList; +import java.util.AbstractList; +import java.util.List; +import java.util.Objects; +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptCost; +import org.apache.calcite.plan.RelOptCostFactory; +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelCollations; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rel.type.RelDataTypeFieldImpl; +import org.apache.calcite.rel.type.RelDataTypeSystem; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeFactoryImpl; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.common.setting.Settings.Key; +import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder; +import org.opensearch.sql.opensearch.storage.OpenSearchIndex; + +@ExtendWith(MockitoExtension.class) +public class CalciteIndexScanCostTest { + static final RelDataTypeFactory typeFactory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); + final RexBuilder builder = new RexBuilder(typeFactory); + + @Mock private static RelOptCluster cluster; + @Mock private static RelOptTable table; + @Mock private static OpenSearchIndex osIndex; + @Mock private static RelOptPlanner planner; + @Mock private static RelMetadataQuery mq; + + @BeforeEach + void setUp() { + RelTraitSet traitSet = mock(RelTraitSet.class); + when(cluster.traitSetOf(any(Convention.class))).thenReturn(traitSet); + when(osIndex.getMaxResultWindow()).thenReturn(10000); + Settings settings = mock(Settings.class); + when(settings.getSettingValue(Key.CALCITE_PUSHDOWN_ROWCOUNT_ESTIMATION_FACTOR)).thenReturn(0.9); + when(osIndex.getSettings()).thenReturn(settings); + + RelOptCostFactory costFactory = mock(RelOptCostFactory.class); + when(planner.getCostFactory()).thenReturn(costFactory); + when(costFactory.makeCost(anyDouble(), anyDouble(), anyDouble())) + .thenAnswer( + invocation -> { + Object[] args = invocation.getArguments(); + RelOptCost optCost = mock(RelOptCost.class); + when(optCost.getRows()).thenReturn((Double) args[0]); + return optCost; + }); + } + + @Test + void test_cost_on_non_pushdown() { + RelDataType relDataType = mock(RelDataType.class); + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(10)); + lenient().when(table.getRowType()).thenReturn(relDataType); + CalciteLogicalIndexScan scan = new CalciteLogicalIndexScan(cluster, table, osIndex); + assertEquals(90000, scan.computeSelfCost(planner, mq).getRows()); + } + + @Test + void test_cost_on_project_pushdown() { + RelDataType relDataType = mock(RelDataType.class); + lenient().when(table.getRowType()).thenReturn(relDataType); + CalciteLogicalIndexScan scan = new CalciteLogicalIndexScan(cluster, table, osIndex); + + List projectDigest = List.of("A"); + scan.getPushDownContext() + .add( + new PushDownOperation( + PushDownType.PROJECT, projectDigest, (OSRequestBuilderAction) req -> {})); + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(projectDigest.size())); + assertEquals(9000, Objects.requireNonNull(scan.computeSelfCost(planner, mq)).getRows()); + + projectDigest = List.of("A", "B", "C"); + scan.getPushDownContext() + .add( + new PushDownOperation( + PushDownType.PROJECT, projectDigest, (OSRequestBuilderAction) req -> {})); + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(projectDigest.size())); + assertEquals(27000, Objects.requireNonNull(scan.computeSelfCost(planner, mq)).getRows()); + } + + @Test + void test_cost_on_limit_pushdown() { + RelDataType relDataType = mock(RelDataType.class); + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(10)); + lenient().when(table.getRowType()).thenReturn(relDataType); + + CalciteLogicalIndexScan scan = new CalciteLogicalIndexScan(cluster, table, osIndex); + + LimitDigest limitDigest = new LimitDigest(100, 0); + scan.getPushDownContext() + .add( + new PushDownOperation( + PushDownType.LIMIT, limitDigest, (OSRequestBuilderAction) req -> {})); + assertEquals(891, Objects.requireNonNull(scan.computeSelfCost(planner, mq)).getRows()); + } + + @Test + void test_cost_on_filter_pushdown() { + RelDataType relDataType = mock(RelDataType.class); + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(10)); + lenient().when(table.getRowType()).thenReturn(relDataType); + + CalciteLogicalIndexScan scan = new CalciteLogicalIndexScan(cluster, table, osIndex); + + RexNode condition = + builder.makeCall( + SqlStdOperatorTable.EQUALS, + builder.makeInputRef(scan, 0), + builder.makeLiteral("Hello")); + FilterDigest filterDigest = new FilterDigest(0, condition); + scan.getPushDownContext() + .add( + new PushDownOperation( + PushDownType.FILTER, filterDigest, (OSRequestBuilderAction) req -> {})); + assertEquals(13500, Objects.requireNonNull(scan.computeSelfCost(planner, mq)).getRows()); + } + + @Test + void test_cost_on_filter_script_pushdown() { + RelDataType relDataType = mock(RelDataType.class); + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(10)); + lenient().when(table.getRowType()).thenReturn(relDataType); + + CalciteLogicalIndexScan scan = new CalciteLogicalIndexScan(cluster, table, osIndex); + + RexNode condition = + builder.makeCall( + SqlStdOperatorTable.EQUALS, + builder.makeInputRef(scan, 0), + builder.makeLiteral("Hello")); + FilterDigest filterDigest = new FilterDigest(1, condition); + scan.getPushDownContext() + .add( + new PushDownOperation( + PushDownType.SCRIPT, filterDigest, (OSRequestBuilderAction) req -> {})); + assertEquals(14985, Objects.requireNonNull(scan.computeSelfCost(planner, mq)).getRows()); + } + + @Test + void test_cost_on_sort_pushdown() { + RelDataType relDataType = mock(RelDataType.class); + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(10)); + lenient().when(table.getRowType()).thenReturn(relDataType); + + CalciteLogicalIndexScan scan = new CalciteLogicalIndexScan(cluster, table, osIndex); + scan.getPushDownContext() + .add(new PushDownOperation(PushDownType.SORT, null, (OSRequestBuilderAction) req -> {})); + assertEquals(99000, Objects.requireNonNull(scan.computeSelfCost(planner, mq)).getRows()); + } + + @Test + void test_cost_on_collapse_pushdown() { + RelDataType relDataType = mock(RelDataType.class); + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(10)); + lenient().when(table.getRowType()).thenReturn(relDataType); + + CalciteLogicalIndexScan scan = new CalciteLogicalIndexScan(cluster, table, osIndex); + scan.getPushDownContext() + .add( + new PushDownOperation(PushDownType.COLLAPSE, null, (OSRequestBuilderAction) req -> {})); + assertEquals(9900, Objects.requireNonNull(scan.computeSelfCost(planner, mq)).getRows()); + } + + @Test + void test_cost_on_aggregate_pushdown() { + RelDataType relDataType = mock(RelDataType.class); + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(10)); + lenient().when(relDataType.getFieldCount()).thenReturn(10); + lenient().when(cluster.getTypeFactory()).thenReturn(typeFactory); + lenient().when(table.getRowType()).thenReturn(relDataType); + + CalciteLogicalIndexScan scan = new CalciteLogicalIndexScan(cluster, table, osIndex); + Aggregate aggregate = + new LogicalAggregate( + cluster, + cluster.traitSetOf(Convention.NONE), + List.of(), + scan, + ImmutableBitSet.of(0), + null, + List.of()); + when(mq.getRowCount(aggregate)).thenReturn(1000d); + AggPushDownAction action = + new AggPushDownAction(Pair.of(List.of(), null), null, List.of()) { + @Override + public void apply(OpenSearchRequestBuilder requestBuilder) {} + }; + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(1)); + lenient().when(relDataType.getFieldCount()).thenReturn(1); + lenient().when(table.getRowType()).thenReturn(relDataType); + + scan.getPushDownContext() + .add(new PushDownOperation(PushDownType.AGGREGATION, aggregate, action)); + assertEquals(1800, Objects.requireNonNull(scan.computeSelfCost(planner, mq)).getRows()); + } + + @Test + void test_cost_on_aggregate_pushdown_with_one_aggCall() { + RelDataType relDataType = mock(RelDataType.class); + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(10)); + lenient().when(relDataType.getFieldCount()).thenReturn(10); + lenient().when(cluster.getTypeFactory()).thenReturn(typeFactory); + lenient().when(table.getRowType()).thenReturn(relDataType); + + CalciteLogicalIndexScan scan = new CalciteLogicalIndexScan(cluster, table, osIndex); + AggPushDownAction action = + new AggPushDownAction(Pair.of(List.of(), null), null, List.of()) { + @Override + public void apply(OpenSearchRequestBuilder requestBuilder) {} + }; + AggregateCall countCall = + AggregateCall.create( + SqlStdOperatorTable.COUNT, + false, + false, + false, + ImmutableList.of(), + ImmutableList.of(), + -1, + null, + RelCollations.EMPTY, + typeFactory.createSqlType(SqlTypeName.BIGINT), + "cnt"); + Aggregate aggregate = + new LogicalAggregate( + cluster, + cluster.traitSetOf(Convention.NONE), + List.of(), + scan, + ImmutableBitSet.of(0), + null, + List.of(countCall)); + when(mq.getRowCount(aggregate)).thenReturn(1000d); + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(2)); + lenient().when(relDataType.getFieldCount()).thenReturn(2); + lenient().when(table.getRowType()).thenReturn(relDataType); + + scan.getPushDownContext() + .add(new PushDownOperation(PushDownType.AGGREGATION, aggregate, action)); + assertEquals(2812.5, Objects.requireNonNull(scan.computeSelfCost(planner, mq)).getRows()); + } + + @Test + void test_cost_on_aggregate_pushdown_with_two_aggCall() { + RelDataType relDataType = mock(RelDataType.class); + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(10)); + lenient().when(relDataType.getFieldCount()).thenReturn(10); + lenient().when(cluster.getTypeFactory()).thenReturn(typeFactory); + lenient().when(table.getRowType()).thenReturn(relDataType); + + CalciteLogicalIndexScan scan = new CalciteLogicalIndexScan(cluster, table, osIndex); + AggPushDownAction action = + new AggPushDownAction(Pair.of(List.of(), null), null, List.of()) { + @Override + public void apply(OpenSearchRequestBuilder requestBuilder) {} + }; + AggregateCall countCall = + AggregateCall.create( + SqlStdOperatorTable.COUNT, + false, + false, + false, + ImmutableList.of(), + ImmutableList.of(), + -1, + null, + RelCollations.EMPTY, + typeFactory.createSqlType(SqlTypeName.BIGINT), + "cnt"); + AggregateCall sumCall = + AggregateCall.create( + SqlStdOperatorTable.SUM, + false, + false, + false, + ImmutableList.of(), + ImmutableList.of(1), + -1, + null, + RelCollations.EMPTY, + typeFactory.createSqlType(SqlTypeName.BIGINT), + "sum"); + Aggregate aggregate = + new LogicalAggregate( + cluster, + cluster.traitSetOf(Convention.NONE), + List.of(), + scan, + ImmutableBitSet.of(0), + null, + List.of(countCall, sumCall)); + when(mq.getRowCount(aggregate)).thenReturn(1000d); + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(3)); + lenient().when(relDataType.getFieldCount()).thenReturn(3); + lenient().when(table.getRowType()).thenReturn(relDataType); + + scan.getPushDownContext() + .add(new PushDownOperation(PushDownType.AGGREGATION, aggregate, action)); + assertEquals( + 3836.2500429153442, Objects.requireNonNull(scan.computeSelfCost(planner, mq)).getRows()); + } + + @Test + void test_cost_on_aggregate_pushdown_with_one_aggCall_with_script() { + RelDataType relDataType = mock(RelDataType.class); + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(10)); + lenient().when(relDataType.getFieldCount()).thenReturn(10); + lenient().when(cluster.getTypeFactory()).thenReturn(typeFactory); + lenient().when(table.getRowType()).thenReturn(relDataType); + + CalciteLogicalIndexScan scan = new CalciteLogicalIndexScan(cluster, table, osIndex); + AggPushDownAction action = + new AggPushDownAction(Pair.of(List.of(), null), null, List.of()) { + @Override + public void apply(OpenSearchRequestBuilder requestBuilder) {} + + @Override + public long getScriptCount() { + return 1; + } + }; + AggregateCall countCall = + AggregateCall.create( + SqlStdOperatorTable.COUNT, + false, + false, + false, + ImmutableList.of(), + ImmutableList.of(), + -1, + null, + RelCollations.EMPTY, + typeFactory.createSqlType(SqlTypeName.BIGINT), + "cnt"); + Aggregate aggregate = + new LogicalAggregate( + cluster, + cluster.traitSetOf(Convention.NONE), + List.of(), + scan, + ImmutableBitSet.of(0), + null, + List.of(countCall)); + when(mq.getRowCount(aggregate)).thenReturn(1000d); + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(2)); + lenient().when(relDataType.getFieldCount()).thenReturn(2); + lenient().when(table.getRowType()).thenReturn(relDataType); + + scan.getPushDownContext() + .add(new PushDownOperation(PushDownType.AGGREGATION, aggregate, action)); + assertEquals( + 2913.7500643730164, Objects.requireNonNull(scan.computeSelfCost(planner, mq)).getRows()); + } + + @Test + void test_cost_on_project_limit_pushdown() { + RelDataType relDataType = mock(RelDataType.class); + lenient().when(table.getRowType()).thenReturn(relDataType); + CalciteLogicalIndexScan scan = new CalciteLogicalIndexScan(cluster, table, osIndex); + + List projectDigest = List.of("A"); + scan.getPushDownContext() + .add( + new PushDownOperation( + PushDownType.PROJECT, projectDigest, (OSRequestBuilderAction) req -> {})); + LimitDigest limitDigest = new LimitDigest(100, 0); + scan.getPushDownContext() + .add( + new PushDownOperation( + PushDownType.LIMIT, limitDigest, (OSRequestBuilderAction) req -> {})); + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(projectDigest.size())); + assertEquals( + 89.10000000000001, Objects.requireNonNull(scan.computeSelfCost(planner, mq)).getRows()); + + // Reverse the push down sequence won't change the cost + scan = new CalciteLogicalIndexScan(cluster, table, osIndex); + scan.getPushDownContext() + .add( + new PushDownOperation( + PushDownType.LIMIT, limitDigest, (OSRequestBuilderAction) req -> {})); + scan.getPushDownContext() + .add( + new PushDownOperation( + PushDownType.PROJECT, projectDigest, (OSRequestBuilderAction) req -> {})); + assertEquals( + 89.10000000000001, Objects.requireNonNull(scan.computeSelfCost(planner, mq)).getRows()); + } + + @Test + void test_cost_on_multi_operator_pushdown_without_agg() { + RelDataType relDataType = mock(RelDataType.class); + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(10)); + lenient().when(table.getRowType()).thenReturn(relDataType); + CalciteLogicalIndexScan scan = new CalciteLogicalIndexScan(cluster, table, osIndex); + + List projectDigest = List.of("A", "B"); + scan.getPushDownContext() + .add( + new PushDownOperation( + PushDownType.PROJECT, projectDigest, (OSRequestBuilderAction) req -> {})); + RexNode condition = + builder.makeCall( + SqlStdOperatorTable.EQUALS, + builder.makeInputRef(scan, 0), + builder.makeLiteral("Hello")); + FilterDigest filterDigest = new FilterDigest(0, condition); + scan.getPushDownContext() + .add( + new PushDownOperation( + PushDownType.FILTER, filterDigest, (OSRequestBuilderAction) req -> {})); + scan.getPushDownContext() + .add(new PushDownOperation(PushDownType.SORT, null, (OSRequestBuilderAction) req -> {})); + LimitDigest limitDigest = new LimitDigest(100, 0); + scan.getPushDownContext() + .add( + new PushDownOperation( + PushDownType.LIMIT, limitDigest, (OSRequestBuilderAction) req -> {})); + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(projectDigest.size())); + assertEquals(1528.2, Objects.requireNonNull(scan.computeSelfCost(planner, mq)).getRows()); + } + + @Test + void test_cost_on_aggregate_pushdown_along_with_others() { + RelDataType relDataType = mock(RelDataType.class); + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(10)); + lenient().when(relDataType.getFieldCount()).thenReturn(10); + lenient().when(cluster.getTypeFactory()).thenReturn(typeFactory); + lenient().when(table.getRowType()).thenReturn(relDataType); + + CalciteLogicalIndexScan scan = new CalciteLogicalIndexScan(cluster, table, osIndex); + AggPushDownAction action = + new AggPushDownAction(Pair.of(List.of(), null), null, List.of()) { + @Override + public void apply(OpenSearchRequestBuilder requestBuilder) {} + + @Override + public long getScriptCount() { + return 1; + } + }; + AggregateCall countCall = + AggregateCall.create( + SqlStdOperatorTable.COUNT, + false, + false, + false, + ImmutableList.of(), + ImmutableList.of(), + -1, + null, + RelCollations.EMPTY, + typeFactory.createSqlType(SqlTypeName.BIGINT), + "cnt"); + Aggregate aggregate = + new LogicalAggregate( + cluster, + cluster.traitSetOf(Convention.NONE), + List.of(), + scan, + ImmutableBitSet.of(0), + null, + List.of(countCall)); + when(mq.getRowCount(aggregate)).thenReturn(1000d); + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(2)); + lenient().when(relDataType.getFieldCount()).thenReturn(2); + lenient().when(table.getRowType()).thenReturn(relDataType); + + List projectDigest1 = List.of("A", "B"); + scan.getPushDownContext() + .add( + new PushDownOperation( + PushDownType.PROJECT, projectDigest1, (OSRequestBuilderAction) req -> {})); + scan.getPushDownContext() + .add(new PushDownOperation(PushDownType.AGGREGATION, aggregate, action)); + List projectDigest2 = List.of("COUNT"); + scan.getPushDownContext() + .add( + new PushDownOperation( + PushDownType.PROJECT, projectDigest2, (AggregationBuilderAction) req -> {})); + scan.getPushDownContext() + .add(new PushDownOperation(PushDownType.SORT, null, (OSRequestBuilderAction) req -> {})); + LimitDigest limitDigest = new LimitDigest(100, 0); + scan.getPushDownContext() + .add( + new PushDownOperation( + PushDownType.LIMIT, limitDigest, (AggregationBuilderAction) req -> {})); + lenient().when(relDataType.getFieldList()).thenReturn(new MockFieldList(projectDigest2.size())); + assertEquals( + 2102.8500643730163, Objects.requireNonNull(scan.computeSelfCost(planner, mq)).getRows()); + } + + private static class MockFieldList extends AbstractList { + private final int size; + + public MockFieldList(int size) { + this.size = size; + } + + @Override + public RelDataTypeField get(int index) { + RelDataType type = + typeFactory.createSqlType(index == 0 ? SqlTypeName.VARCHAR : SqlTypeName.BIGINT); + return new RelDataTypeFieldImpl("dummy", index, type); + } + + @Override + public int size() { + return size; + } + } +}