diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index eb74465a624..389767d3c08 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -18,6 +18,7 @@ import static org.opensearch.sql.calcite.utils.PlanUtils.ROW_NUMBER_COLUMN_NAME_MAIN; import static org.opensearch.sql.calcite.utils.PlanUtils.ROW_NUMBER_COLUMN_NAME_SUBSEARCH; import static org.opensearch.sql.calcite.utils.PlanUtils.getRelation; +import static org.opensearch.sql.calcite.utils.PlanUtils.getRexCall; import static org.opensearch.sql.calcite.utils.PlanUtils.transformPlanToAttachChild; import com.google.common.base.Strings; @@ -53,6 +54,7 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.rex.RexWindowBounds; +import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; @@ -813,6 +815,23 @@ private void projectPlusOverriding( context.relBuilder.rename(expectedRenameFields); } + private List> extractInputRefList(List aggCalls) { + return aggCalls.stream() + .map(RelBuilder.AggCall::over) + .map(RelBuilder.OverCall::toRex) + .map(node -> getRexCall(node, this::isCountField)) + .map(list -> list.isEmpty() ? null : list.get(0)) + .map(PlanUtils::getInputRefs) + .collect(Collectors.toList()); + } + + /** Is count(FIELD) */ + private boolean isCountField(RexCall call) { + return call.isA(SqlKind.COUNT) + && call.getOperands().size() == 1 // count(FIELD) + && call.getOperands().get(0) instanceof RexInputRef; + } + /** * Resolve the aggregation with trimming unused fields to avoid bugs in {@link * org.apache.calcite.sql2rel.RelDecorrelator#decorrelateRel(Aggregate, boolean)} @@ -826,6 +845,74 @@ private Pair, List> aggregateWithTrimming( List groupExprList, List aggExprList, CalcitePlanContext context) { + Pair, List> resolved = + resolveAttributesForAggregation(groupExprList, aggExprList, context); + List resolvedGroupByList = resolved.getLeft(); + List resolvedAggCallList = resolved.getRight(); + + // `doc_count` optimization required a filter `isNotNull(RexInputRef)` for the + // `count(FIELD)` aggregation which only can be applied to single FIELD without grouping: + // + // Example 1: source=t | stats count(a) + // Before: Aggregate(count(a)) + // \- Scan t + // After: Aggregate(count(a)) + // \- Filter(isNotNull(a)) + // \- Scan t + // + // Example 2: source=t | stats count(a), count(a) + // Before: Aggregate(count(a), count(a)) + // \- Scan t + // After: Aggregate(count(a), count(a)) + // \- Filter(isNotNull(a)) + // \- Scan t + // + // Example 3: source=t | stats count(a) by b + // Before & After: Aggregate(count(a) by b) + // \- Scan t + // + // Example 4: source=t | stats count() + // Before & After: Aggregate(count()) + // \- Scan t + // + // Example 5: source=t | stats count(), count(a) + // Before & After: Aggregate(count(), count(a)) + // \- Scan t + // + // Example 6: source=t | stats count(a), count(b) + // Before & After: Aggregate(count(a), count(b)) + // \- Scan t + // + // Example 7: source=t | stats count(a+1) + // Before & After: Aggregate(count(a+1)) + // \- Scan t + if (resolvedGroupByList.isEmpty()) { + List> refsPerCount = extractInputRefList(resolvedAggCallList); + List distinctRefsOfCounts; + if (context.relBuilder.peek() instanceof org.apache.calcite.rel.core.Project) { + org.apache.calcite.rel.core.Project project = + (org.apache.calcite.rel.core.Project) context.relBuilder.peek(); + List mappedInProject = + refsPerCount.stream() + .flatMap(List::stream) + .map(ref -> project.getProjects().get(ref.getIndex())) + .collect(Collectors.toList()); + if (mappedInProject.stream().allMatch(RexInputRef.class::isInstance)) { + distinctRefsOfCounts = + mappedInProject.stream().map(RexInputRef.class::cast).distinct().collect(Collectors.toList()); + } else { + distinctRefsOfCounts = List.of(); + } + } else { + distinctRefsOfCounts = refsPerCount.stream().flatMap(List::stream).distinct().collect(Collectors.toList()); + } + if (distinctRefsOfCounts.size() == 1 && refsPerCount.stream().noneMatch(List::isEmpty)) { + context.relBuilder.filter(context.relBuilder.isNotNull(distinctRefsOfCounts.get(0))); + } + } + + // Add project before aggregate: + // // Example 1: source=t | where a > 1 | stats avg(b + 1) by c // Before: Aggregate(avg(b + 1)) // \- Filter(a > 1) @@ -836,23 +923,22 @@ private Pair, List> aggregateWithTrimming( // \- Scan t // // Example 2: source=t | where a > 1 | top b by c - // Before: Aggregate(count) - // \-Filter(a > 1) + // Before: Aggregate(count(b) by c) + // \-Filter(a > 1 && isNotNull(b)) // \- Scan t - // After: Aggregate(count) + // After: Aggregate(count(b) by c) // \- Project([c, b]) - // \- Filter(a > 1) + // \- Filter(a > 1 && isNotNull(b)) // \- Scan t - // Example 3: source=t | stats count(): no project added for count() - // Before: Aggregate(count) + // + // Example 3: source=t | stats count(): no change for count() + // Before: Aggregate(count()) // \- Scan t - // After: Aggregate(count) + // After: Aggregate(count()) // \- Scan t - Pair, List> resolved = - resolveAttributesForAggregation(groupExprList, aggExprList, context); List trimmedRefs = new ArrayList<>(); - trimmedRefs.addAll(PlanUtils.getInputRefs(resolved.getLeft())); // group-by keys first - trimmedRefs.addAll(PlanUtils.getInputRefsFromAggCall(resolved.getRight())); + trimmedRefs.addAll(PlanUtils.getInputRefs(resolvedGroupByList)); // group-by keys first + trimmedRefs.addAll(PlanUtils.getInputRefsFromAggCall(resolvedAggCallList)); context.relBuilder.project(trimmedRefs); // Re-resolve all attributes based on adding trimmed Project. diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java index 6a206dc96fe..ead88c2e453 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java @@ -15,6 +15,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.function.Predicate; import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.calcite.plan.RelOptTable; @@ -282,6 +283,9 @@ static RelBuilder.AggCall makeAggCall( /** Get all uniq input references from a RexNode. */ static List getInputRefs(RexNode node) { + if (node == null) { + return List.of(); + } List inputRefs = new ArrayList<>(); node.accept( new RexVisitorImpl(true) { @@ -301,6 +305,26 @@ static List getInputRefs(List nodes) { return nodes.stream().flatMap(node -> getInputRefs(node).stream()).collect(Collectors.toList()); } + /** Get all uniq RexCall from RexNode with a predicate */ + static List getRexCall(RexNode node, Predicate predicate) { + List list = new ArrayList<>(); + node.accept( + new RexVisitorImpl(true) { + @Override + public Void visitCall(RexCall inputCall) { + if (predicate.test(inputCall)) { + if (!list.contains(inputCall)) { + list.add(inputCall); + } + } else { + inputCall.getOperands().forEach(call -> call.accept(this)); + } + return null; + } + }); + return list; + } + /** Get all uniq input references from a list of agg calls. */ static List getInputRefsFromAggCall(List aggCalls) { return aggCalls.stream() diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 4ce1e96ebc1..f4153149ff5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.calcite.remote; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_LOGS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_SIMPLE; @@ -643,6 +644,143 @@ public void testExplainMinOnStringField() throws IOException { explainQueryToString("source=opensearch-sql_test_index_account | stats min(firstname)")); } + @Test + @Override + public void testCountAggPushDownExplain() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + // should be optimized by hits.total.value + String expected = loadExpectedPlan("explain_count_agg_push1.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString("source=opensearch-sql_test_index_account | stats count() as cnt")); + + // should be optimized + expected = loadExpectedPlan("explain_count_agg_push2.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account | stats count(lastname) as cnt")); + + // should be optimized + expected = loadExpectedPlan("explain_count_agg_push3.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account | eval name = lastname | stats count(name) as" + + " cnt")); + + // should be optimized + expected = loadExpectedPlan("explain_count_agg_push4.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account | stats count() as c1, count() as c2")); + + // should be optimized + expected = loadExpectedPlan("explain_count_agg_push5.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account | stats count(lastname) as c1," + + " count(lastname) as c2")); + + // should be optimized + expected = loadExpectedPlan("explain_count_agg_push6.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account | eval name = lastname | stats" + + " count(lastname), count(name)")); + + // should not be optimized + expected = loadExpectedPlan("explain_count_agg_push7.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account | stats count(balance + 1) as cnt")); + + // should not be optimized + expected = loadExpectedPlan("explain_count_agg_push8.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account | stats count() as c1, count(lastname) as" + + " c2")); + + // should not be optimized + expected = loadExpectedPlan("explain_count_agg_push9.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account | stats count(firstname), count(lastname)")); + + // should not be optimized + expected = loadExpectedPlan("explain_count_agg_push10.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account | eval name = lastname | stats" + + " count(firstname), count(name)")); + } + + @Test + public void testExplainCountsByAgg() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String expected = loadExpectedPlan("explain_agg_counts_by1.yaml"); + // case of only count(): doc_count works + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + String.format( + "source=%s | stats count(), count() as c1 by gender", TEST_INDEX_ACCOUNT))); + + // count(FIELD) by: doc_count doesn't work + expected = loadExpectedPlan("explain_agg_counts_by2.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + String.format( + "source=%s | stats count(balance) as c1, count(balance) as c2 by gender", + TEST_INDEX_ACCOUNT))); + + // count(FIELD) by: doc_count doesn't work + expected = loadExpectedPlan("explain_agg_counts_by3.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + String.format( + "source=%s | eval account_number_alias = account_number" + + " | stats count(account_number), count(account_number_alias) as c2 by gender", + TEST_INDEX_ACCOUNT))); + + // count() + count(FIELD)): doc_count doesn't work + expected = loadExpectedPlan("explain_agg_counts_by4.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + String.format( + "source=%s | stats count(), count(account_number) by gender", TEST_INDEX_ACCOUNT))); + + // count(FIELD1) + count(FIELD2)) by: doc_count doesn't work + expected = loadExpectedPlan("explain_agg_counts_by5.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + String.format( + "source=%s | stats count(balance), count(account_number) by gender", + TEST_INDEX_ACCOUNT))); + + // case of count(EXPRESSION) by: doc_count doesn't work + expected = loadExpectedPlan("explain_agg_counts_by6.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + String.format( + "source=%s | eval b_1 = balance + 1" + + " | stats count(b_1), count(pow(balance, 2)) as c3 by gender", + TEST_INDEX_ACCOUNT))); + } + @Test public void testExplainSortOnMetricsNoBucketNullable() throws IOException { // TODO enhancement later: https://github.com/opensearch-project/sql/issues/4282 diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java index c6f6875d989..2f232ad8749 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java @@ -764,4 +764,29 @@ public void testStatsBySpanTimeWithNullBucket() throws IOException { rows(7931, "2025-07-31 00:00:00"), rows(8086, "2025-07-31 12:00:00")); } + + @Test + public void testStatsByCounts() throws IOException { + JSONObject response = + executeQuery( + String.format( + "source=%s | eval b_1 = balance + 1 | stats count(), count() as c1," + + " count(account_number), count(lastname) as c2, count(balance/10)," + + " count(pow(balance, 2)) as c3, count(b_1) by gender", + TEST_INDEX_ACCOUNT)); + verifySchema( + response, + schema("count()", null, isCalciteEnabled() ? "bigint" : "int"), + schema("c1", null, isCalciteEnabled() ? "bigint" : "int"), + schema("count(account_number)", null, isCalciteEnabled() ? "bigint" : "int"), + schema("c2", null, isCalciteEnabled() ? "bigint" : "int"), + schema("count(balance/10)", null, isCalciteEnabled() ? "bigint" : "int"), + schema("c3", null, isCalciteEnabled() ? "bigint" : "int"), + schema("count(b_1)", null, isCalciteEnabled() ? "bigint" : "int"), + schema("gender", null, "string")); + verifyDataRows( + response, + rows(493, 493, 493, 493, 493, 493, 493, "F"), + rows(507, 507, 507, 507, 507, 507, 507, "M")); + } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml new file mode 100644 index 00000000000..ddeefad911b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], c1=[$1], gender=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(gender=[$4]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..1=[{inputs}], count()=[$t0], count()0=[$t0], gender=[$t1]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->10000, PROJECT->[count(), gender]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml new file mode 100644 index 00000000000..000995f3c2a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(c1=[$1], c2=[$1], gender=[$0]) + LogicalAggregate(group=[{0}], c1=[COUNT($1)]) + LogicalProject(gender=[$4], balance=[$3]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..1=[{inputs}], c1=[$t0], c10=[$t0], gender=[$t1]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c1=COUNT($1)), LIMIT->10000, PROJECT->[c1, gender]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"c1":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by3.yaml new file mode 100644 index 00000000000..a4dfdb25064 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by3.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count(account_number)=[$1], c2=[$2], gender=[$0]) + LogicalAggregate(group=[{0}], count(account_number)=[COUNT($1)], c2=[COUNT($2)]) + LogicalProject(gender=[$4], account_number=[$0], account_number_alias=[$0]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(account_number)=COUNT($1),c2=COUNT($2)), PROJECT->[count(account_number), c2, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(account_number)":{"value_count":{"field":"account_number"}},"c2":{"value_count":{"field":"account_number"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by4.yaml new file mode 100644 index 00000000000..e56eb5ad662 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by4.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], count(account_number)=[$2], gender=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()], count(account_number)=[COUNT($1)]) + LogicalProject(gender=[$4], account_number=[$0]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT(),count(account_number)=COUNT($1)), PROJECT->[count(), count(account_number), gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count()":{"value_count":{"field":"_index"}},"count(account_number)":{"value_count":{"field":"account_number"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by5.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by5.yaml new file mode 100644 index 00000000000..dc7bf3629f2 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by5.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count(balance)=[$1], count(account_number)=[$2], gender=[$0]) + LogicalAggregate(group=[{0}], count(balance)=[COUNT($1)], count(account_number)=[COUNT($2)]) + LogicalProject(gender=[$4], balance=[$3], account_number=[$0]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(balance)=COUNT($1),count(account_number)=COUNT($2)), PROJECT->[count(balance), count(account_number), gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(balance)":{"value_count":{"field":"balance"}},"count(account_number)":{"value_count":{"field":"account_number"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml new file mode 100644 index 00000000000..a385eaf690d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count(b_1)=[$1], c3=[$2], gender=[$0]) + LogicalAggregate(group=[{0}], count(b_1)=[COUNT($1)], c3=[COUNT($2)]) + LogicalProject(gender=[$4], b_1=[+($3, 1)], $f3=[POWER($3, 2)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(b_1)=COUNT($1),c3=COUNT($2)), PROJECT->[count(b_1), c3, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(b_1)":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBAXsKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDMsCiAgICAgICJuYW1lIjogIiQzIgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}},"c3":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBEXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJQT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiAzLAogICAgICAibmFtZSI6ICIkMyIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgAXamF2YS51dGlsLkxpbmtlZEhhc2hNYXA0wE5cEGzA+wIAAVoAC2FjY2Vzc09yZGVyeHIAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAt0AA5hY2NvdW50X251bWJlcn5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd0AAlmaXJzdG5hbWVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AEXhwfnEAfgALdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAx0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAceHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAeAAAAAHNxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgASfnEAfgALdAAGU1RSSU5HfnEAfgAYdAAHS2V5d29yZHEAfgAdeHQAB2FkZHJlc3NzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAAAeHQAB2JhbGFuY2VxAH4ADXQABmdlbmRlcnNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQABGNpdHlzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AAhlbXBsb3llcnNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQABXN0YXRlc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAADYWdlcQB+AA10AAVlbWFpbHNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGxhc3RuYW1lc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4eAB4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics1.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics1.json index 3c9b278aa97..7fa93a13cf4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics1.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics1.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], dir0=[ASC-nulls-first])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n LogicalFilter(condition=[IS NOT NULL($7)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($7), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"state\",\"boost\":1.0}},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":false,\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($7), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"state\",\"boost\":1.0}},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":false,\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics2.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics2.json index 9ce19094fdc..fb2b1104bd9 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics2.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics2.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], dir0=[ASC-nulls-first])\n LogicalProject(count()=[$2], gender=[$0], state=[$1])\n LogicalAggregate(group=[{0, 1}], count()=[COUNT()])\n LogicalProject(gender=[$4], state=[$7])\n LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->AND(IS NOT NULL($4), IS NOT NULL($7)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), gender, state]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"must\":[{\"exists\":{\"field\":\"gender\",\"boost\":1.0}},{\"exists\":{\"field\":\"state\",\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":false,\"order\":\"asc\"}}},{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":false,\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->AND(IS NOT NULL($4), IS NOT NULL($7)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), gender, state]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"must\":[{\"exists\":{\"field\":\"gender\",\"boost\":1.0}},{\"exists\":{\"field\":\"state\",\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":false,\"order\":\"asc\"}}},{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":false,\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_append_command.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_append_command.json index 44673d64b6e..7a857107a13 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_append_command.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_append_command.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalUnion(all=[true])\n LogicalProject(cnt=[$1], gender=[$0])\n LogicalAggregate(group=[{0}], cnt=[COUNT($1)])\n LogicalProject(gender=[$4], balance=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n LogicalProject(cnt=[$0], gender=[null:VARCHAR])\n LogicalAggregate(group=[{}], cnt=[COUNT()])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableUnion(all=[true])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},cnt=COUNT($1)), PROJECT->[cnt, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"cnt\":{\"value_count\":{\"field\":\"balance\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableCalc(expr#0=[{inputs}], expr#1=[null:VARCHAR], proj#0..1=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"cnt\":{\"value_count\":{\"field\":\"_index\"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableUnion(all=[true])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},cnt=COUNT($1)), PROJECT->[cnt, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"cnt\":{\"value_count\":{\"field\":\"balance\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableCalc(expr#0=[{inputs}], expr#1=[null:VARCHAR], proj#0..1=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"track_total_hits\":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push.json deleted file mode 100644 index 63b4c6d8ab2..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], cnt=[COUNT()])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"cnt\":{\"value_count\":{\"field\":\"_index\"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push1.yaml new file mode 100644 index 00000000000..e0d0a3d0070 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push1.yaml @@ -0,0 +1,7 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], cnt=[COUNT()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push10.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push10.yaml new file mode 100644 index 00000000000..26ab37adc53 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push10.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], count(firstname)=[COUNT($0)], count(name)=[COUNT($1)]) + LogicalProject(firstname=[$1], name=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count(firstname)=COUNT($0),count(name)=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"count(firstname)":{"value_count":{"field":"firstname.keyword"}},"count(name)":{"value_count":{"field":"lastname.keyword"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push2.yaml new file mode 100644 index 00000000000..6a2d697a071 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push2.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], cnt=[COUNT($0)]) + LogicalProject(lastname=[$10]) + LogicalFilter(condition=[IS NOT NULL($10)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($10), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"lastname","boost":1.0}},"sort":[],"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push3.yaml new file mode 100644 index 00000000000..71be9d237c1 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push3.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], cnt=[COUNT($0)]) + LogicalProject(name=[$17]) + LogicalFilter(condition=[IS NOT NULL($10)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], name=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($10), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"lastname","boost":1.0}},"sort":[],"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push4.yaml new file mode 100644 index 00000000000..b191f90b170 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push4.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(c1=[$0], c2=[$0]) + LogicalAggregate(group=[{}], c1=[COUNT()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0=[{inputs}], c1=[$t0], c2=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},c1=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push5.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push5.yaml new file mode 100644 index 00000000000..689138f358d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push5.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(c1=[$0], c2=[$0]) + LogicalAggregate(group=[{}], c1=[COUNT($0)]) + LogicalProject(lastname=[$10]) + LogicalFilter(condition=[IS NOT NULL($10)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0=[{inputs}], c1=[$t0], c2=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($10), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},c1=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"lastname","boost":1.0}},"sort":[],"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push6.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push6.yaml new file mode 100644 index 00000000000..82260e27bbc --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push6.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], count(lastname)=[COUNT($0)], count(name)=[COUNT($1)]) + LogicalProject(lastname=[$10], name=[$17]) + LogicalFilter(condition=[IS NOT NULL($10)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], name=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($10), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count(lastname)=COUNT($0),count(name)=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"lastname","boost":1.0}},"sort":[],"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml new file mode 100644 index 00000000000..041d862fbc4 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], cnt=[COUNT($0)]) + LogicalProject($f1=[+($3, 1)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"cnt":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBAXsKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDMsCiAgICAgICJuYW1lIjogIiQzIgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push8.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push8.yaml new file mode 100644 index 00000000000..beaac7cbe69 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push8.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], c1=[COUNT()], c2=[COUNT($0)]) + LogicalProject(lastname=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},c1=COUNT(),c2=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"c1":{"value_count":{"field":"_index"}},"c2":{"value_count":{"field":"lastname.keyword"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push9.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push9.yaml new file mode 100644 index 00000000000..b5548293b7f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push9.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], count(firstname)=[COUNT($0)], count(lastname)=[COUNT($1)]) + LogicalProject(firstname=[$1], lastname=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count(firstname)=COUNT($0),count(lastname)=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"count(firstname)":{"value_count":{"field":"firstname.keyword"}},"count(lastname)":{"value_count":{"field":"lastname.keyword"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown.json index ada9de0f0d8..5fbc3abecf1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown2.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown2.json index db889cf0c94..391568b1234 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown2.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown2.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(fetch=[100])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->100, PROJECT->[count(), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":100,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->100, PROJECT->[count(), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":100,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown3.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown3.json index 3432ea267a7..986459fd2f4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown3.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown3.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(offset=[10], fetch=[10])\n LogicalSort(fetch=[100])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":20,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":20,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.json index 8ffece29593..635a33b5d6e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(offset=[10], fetch=[10])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[100])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT->[0 ASC FIRST], LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":20,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT->[0 ASC FIRST], LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":20,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown5.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown5.json index f18f0faa08c..e9756e74db6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown5.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown5.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(offset=[10], fetch=[10])\n LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[100])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableLimit(offset=[10], fetch=[10])\n EnumerableLimit(fetch=[100])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableLimit(offset=[10], fetch=[10])\n EnumerableLimit(fetch=[100])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.json index f5eea7334b3..2db9c47af12 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(offset=[10], fetch=[10])\n LogicalSort(fetch=[100])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n LogicalFilter(condition=[IS NOT NULL($7)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($7), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"state\",\"boost\":1.0}},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":20,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":false,\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($7), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"state\",\"boost\":1.0}},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":20,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":false,\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.json index bec9b909307..40adc108671 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(offset=[10], fetch=[10])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[100])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n LogicalFilter(condition=[IS NOT NULL($7)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($7), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT->[0 ASC FIRST], LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"state\",\"boost\":1.0}},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":20,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":false,\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($7), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT->[0 ASC FIRST], LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"state\",\"boost\":1.0}},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":20,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":false,\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.json index f4e9e733e41..2f13054be6e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], address_length=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(address_length=[CHAR_LENGTH($2)])\n LogicalFilter(condition=[>(CHAR_LENGTH($2), 0)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[address], SCRIPT->>(CHAR_LENGTH($0), 0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), address_length], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJhZGRyZXNzIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiPiIsCiAgICAia2luZCI6ICJHUkVBVEVSX1RIQU4iLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAgICAgImtpbmQiOiAiQ0hBUl9MRU5HVEgiLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgIm5hbWUiOiAiJDAiCiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAdhZGRyZXNzc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAAAeHh4\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},\"_source\":{\"includes\":[\"address\"],\"excludes\":[]},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"address_length\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJhZGRyZXNzIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AKZ7CiAgIm9wIjogewogICAgIm5hbWUiOiAiQ0hBUl9MRU5HVEgiLAogICAgImtpbmQiOiAiQ0hBUl9MRU5HVEgiLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[address], SCRIPT->>(CHAR_LENGTH($0), 0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), address_length], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJhZGRyZXNzIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiPiIsCiAgICAia2luZCI6ICJHUkVBVEVSX1RIQU4iLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAgICAgImtpbmQiOiAiQ0hBUl9MRU5HVEgiLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgIm5hbWUiOiAiJDAiCiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAdhZGRyZXNzc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAAAeHh4\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},\"_source\":{\"includes\":[\"address\"],\"excludes\":[]},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"address_length\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJhZGRyZXNzIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AKZ7CiAgIm9wIjogewogICAgIm5hbWUiOiAiQ0hBUl9MRU5HVEgiLAogICAgImtpbmQiOiAiQ0hBUl9MRU5HVEgiLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span.json index 3841efb47d3..01258e3fd0b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(age,10)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(age,10)=[SPAN($10, 10, null:NULL)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(age,10)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(age,10)\":{\"histogram\":{\"field\":\"age\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\",\"interval\":10.0}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(age,10)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(age,10)\":{\"histogram\":{\"field\":\"age\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\",\"interval\":10.0}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span_non_bucket_nullable.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span_non_bucket_nullable.json index d6373bf58a9..779e12361b8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span_non_bucket_nullable.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span_non_bucket_nullable.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(age,10)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(age,10)=[SPAN($10, 10, null:NULL)])\n LogicalFilter(condition=[IS NOT NULL($10)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age], FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(age,10)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"age\",\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(age,10)\":{\"histogram\":{\"field\":\"age\",\"missing_bucket\":false,\"order\":\"asc\",\"interval\":10.0}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age], FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(age,10)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"age\",\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(age,10)\":{\"histogram\":{\"field\":\"age\",\"missing_bucket\":false,\"order\":\"asc\",\"interval\":10.0}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.json index a27724a1cd8..e44d3d9c613 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(birthdate,1m)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(birthdate,1m)=[SPAN($3, 1, 'm')])\n LogicalFilter(condition=[IS NOT NULL($3)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[birthdate], FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1m)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"birthdate\",\"boost\":1.0}},\"_source\":{\"includes\":[\"birthdate\"],\"excludes\":[]},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(birthdate,1m)\":{\"date_histogram\":{\"field\":\"birthdate\",\"missing_bucket\":false,\"order\":\"asc\",\"fixed_interval\":\"1m\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[birthdate], FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1m)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"birthdate\",\"boost\":1.0}},\"_source\":{\"includes\":[\"birthdate\"],\"excludes\":[]},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(birthdate,1m)\":{\"date_histogram\":{\"field\":\"birthdate\",\"missing_bucket\":false,\"order\":\"asc\",\"fixed_interval\":\"1m\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.json index 9a306d02f12..774863863c2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(birthdate,1M)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(birthdate,1M)=[SPAN($3, 1, 'M')])\n LogicalFilter(condition=[IS NOT NULL($3)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[birthdate], FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1M)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"birthdate\",\"boost\":1.0}},\"_source\":{\"includes\":[\"birthdate\"],\"excludes\":[]},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(birthdate,1M)\":{\"date_histogram\":{\"field\":\"birthdate\",\"missing_bucket\":false,\"order\":\"asc\",\"calendar_interval\":\"1M\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[birthdate], FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1M)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"birthdate\",\"boost\":1.0}},\"_source\":{\"includes\":[\"birthdate\"],\"excludes\":[]},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(birthdate,1M)\":{\"date_histogram\":{\"field\":\"birthdate\",\"missing_bucket\":false,\"order\":\"asc\",\"calendar_interval\":\"1M\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java index cdc3d4462fe..8978cc4a973 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java @@ -76,6 +76,13 @@ public interface OpenSearchClient { */ Map meta(); + /** + * Force to clean up resources related to the search request. + * + * @param request search request + */ + void forceCleanup(OpenSearchRequest request); + /** * Clean up resources related to the search request, for example scroll context. * diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index 40ed517c223..a3618c8717b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -171,6 +171,27 @@ public Map meta() { client.settings().get("plugins.sql.pagination.api", "true")); } + @Override + public void forceCleanup(OpenSearchRequest request) { + if (request instanceof OpenSearchScrollRequest) { + request.forceClean( + scrollId -> { + try { + client.prepareClearScroll().addScrollId(scrollId).get(); + } catch (Exception e) { + throw new IllegalStateException( + "Failed to clean up resources for search request " + request, e); + } + }); + } else { + request.forceClean( + pitId -> { + DeletePitRequest deletePitRequest = new DeletePitRequest(pitId); + deletePit(deletePitRequest); + }); + } + } + @Override public void cleanup(OpenSearchRequest request) { if (request instanceof OpenSearchScrollRequest) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index 2551626ebdb..c28f28b0505 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -180,6 +180,29 @@ public Map meta() { } } + @Override + public void forceCleanup(OpenSearchRequest request) { + if (request instanceof OpenSearchScrollRequest) { + request.forceClean( + scrollId -> { + try { + ClearScrollRequest clearRequest = new ClearScrollRequest(); + clearRequest.addScrollId(scrollId); + client.clearScroll(clearRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + throw new IllegalStateException( + "Failed to clean up resources for search request " + request, e); + } + }); + } else { + request.forceClean( + pitId -> { + DeletePitRequest deletePitRequest = new DeletePitRequest(pitId); + deletePit(deletePitRequest); + }); + } + } + @Override public void cleanup(OpenSearchRequest request) { if (request instanceof OpenSearchScrollRequest) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchIndexScanRule.java index 50f67485cca..24abb3c3bc9 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchIndexScanRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchIndexScanRule.java @@ -15,6 +15,7 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexOver; +import org.apache.calcite.util.Pair; import org.opensearch.sql.opensearch.storage.OpenSearchIndex; import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; @@ -41,8 +42,10 @@ static boolean isLimitPushed(AbstractCalciteIndexScan scan) { // e.g. Project($0, $0, +($0,1)). We cannot push down the Aggregate for this corner case. // TODO: Simplify the Project where there is RexCall by adding a new rule. static boolean distinctProjectList(LogicalProject project) { - Set rexSet = new HashSet<>(); - return project.getProjects().stream().allMatch(rexSet::add); + // Change to Set> to resolve + // https://github.com/opensearch-project/sql/issues/4347 + Set> rexSet = new HashSet<>(); + return project.getNamedProjects().stream().allMatch(rexSet::add); } static boolean containsRexOver(LogicalProject project) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java index 6023fd8f4ce..66ac901a680 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java @@ -64,6 +64,7 @@ import org.opensearch.search.aggregations.metrics.ExtendedStats; import org.opensearch.search.aggregations.metrics.PercentilesAggregationBuilder; import org.opensearch.search.aggregations.metrics.TopHitsAggregationBuilder; +import org.opensearch.search.aggregations.metrics.ValueCountAggregationBuilder; import org.opensearch.search.aggregations.support.ValueType; import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.opensearch.search.sort.SortOrder; @@ -77,6 +78,7 @@ import org.opensearch.sql.opensearch.request.PredicateAnalyzer.NamedFieldExpression; import org.opensearch.sql.opensearch.response.agg.ArgMaxMinParser; import org.opensearch.sql.opensearch.response.agg.CompositeAggregationParser; +import org.opensearch.sql.opensearch.response.agg.CountAsTotalHitsParser; import org.opensearch.sql.opensearch.response.agg.MetricParser; import org.opensearch.sql.opensearch.response.agg.NoBucketAggregationParser; import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; @@ -201,19 +203,64 @@ public static Pair, OpenSearchAggregationResponseParser Builder metricBuilder = builderAndParser.getLeft(); List metricParserList = builderAndParser.getRight(); + // both count() and count(FIELD) can apply doc_count optimization in non-bucket aggregation, + // but only count() can apply doc_count optimization in bucket aggregation. + boolean countAllOnly = !aggregate.getGroupSet().isEmpty(); + Pair, Builder> pair = + removeCountAggregationBuilders(metricBuilder, countAllOnly); + List removedCountAggBuilders = pair.getLeft(); + Builder newMetricBuilder = pair.getRight(); + + boolean removedCountAggBuildersHaveSomeField = + removedCountAggBuilders.stream() + .map(ValuesSourceAggregationBuilder::fieldName) + .distinct() + .count() + == 1; + boolean allCountAggRemoved = + removedCountAggBuilders.size() == metricBuilder.getAggregatorFactories().size(); if (aggregate.getGroupSet().isEmpty()) { - return Pair.of( - ImmutableList.copyOf(metricBuilder.getAggregatorFactories()), - new NoBucketAggregationParser(metricParserList)); + if (allCountAggRemoved && removedCountAggBuildersHaveSomeField) { + // The optimization must require all count aggregations are removed, + // and they have only one field name + List countAggNameList = + removedCountAggBuilders.stream() + .map(ValuesSourceAggregationBuilder::getName) + .collect(Collectors.toList()); + return Pair.of( + ImmutableList.copyOf(newMetricBuilder.getAggregatorFactories()), + new CountAsTotalHitsParser(countAggNameList)); + } else { + return Pair.of( + ImmutableList.copyOf(metricBuilder.getAggregatorFactories()), + new NoBucketAggregationParser(metricParserList)); + } } else { List> buckets = createCompositeBuckets(groupList, project, helper); + AggregationBuilder aggregationBuilder = + AggregationBuilders.composite("composite_buckets", buckets) + .size(AGGREGATION_BUCKET_SIZE); + + // For bucket aggregation, no count() aggregator or not all aggregators are count(), + // fallback to original ValueCountAggregation. + if (removedCountAggBuilders.isEmpty() + || removedCountAggBuilders.size() != metricBuilder.getAggregatorFactories().size()) { + aggregationBuilder.subAggregations(metricBuilder); + return Pair.of( + Collections.singletonList(aggregationBuilder), + new CompositeAggregationParser(metricParserList)); + } + // No need to register sub-factories if no aggregator factories left after removing all + // ValueCountAggregationBuilder. + if (!newMetricBuilder.getAggregatorFactories().isEmpty()) { + aggregationBuilder.subAggregations(newMetricBuilder); + } + List countAggNameList = + removedCountAggBuilders.stream().map(ValuesSourceAggregationBuilder::getName).collect(Collectors.toList()); return Pair.of( - Collections.singletonList( - AggregationBuilders.composite("composite_buckets", buckets) - .subAggregations(metricBuilder) - .size(AGGREGATION_BUCKET_SIZE)), - new CompositeAggregationParser(metricParserList)); + Collections.singletonList(aggregationBuilder), + new CompositeAggregationParser(metricParserList, countAggNameList)); } } catch (Throwable e) { Throwables.throwIfInstanceOf(e, UnsupportedOperationException.class); @@ -221,6 +268,30 @@ public static Pair, OpenSearchAggregationResponseParser } } + /** + * Remove all ValueCountAggregationBuilder from metric builder, and return the removed + * ValueCountAggregationBuilder list. + * + * @param metricBuilder metrics builder + * @param countAllOnly remove count() only, or count(FIELD) will be removed. + * @return a pair of removed ValueCountAggregationBuilder and updated metric builder + */ + private static Pair, Builder> removeCountAggregationBuilders( + Builder metricBuilder, boolean countAllOnly) { + List countAggregatorFactories = + metricBuilder.getAggregatorFactories().stream() + .filter(ValueCountAggregationBuilder.class::isInstance) + .map(ValueCountAggregationBuilder.class::cast) + .filter(vc -> vc.script() == null) + .filter(vc -> !countAllOnly || vc.fieldName().equals("_index")) + .collect(Collectors.toList()); + List copy = new ArrayList<>(metricBuilder.getAggregatorFactories()); + copy.removeAll(countAggregatorFactories); + Builder newMetricBuilder = new AggregatorFactories.Builder(); + copy.forEach(newMetricBuilder::addAggregator); + return Pair.of(countAggregatorFactories, newMetricBuilder); + } + private static Pair> processAggregateCalls( List aggFieldNames, List aggCalls, diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java index 77ceff1d564..c6a3e6197db 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java @@ -119,6 +119,14 @@ public OpenSearchQueryRequest( this.pitId = pitId; } + /** true if the request is a count aggregation request. */ + public boolean isCountAggRequest() { + return !searchDone + && sourceBuilder.size() == 0 + && sourceBuilder.trackTotalHitsUpTo() != null // only set in v3 + && sourceBuilder.trackTotalHitsUpTo() == Integer.MAX_VALUE; + } + /** * Constructs OpenSearchQueryRequest from serialized representation. * @@ -163,14 +171,18 @@ public OpenSearchResponse search( if (this.pitId == null) { // When SearchRequest doesn't contain PitId, fetch single page request if (searchDone) { - return new OpenSearchResponse(SearchHits.empty(), exprValueFactory, includes); + return new OpenSearchResponse( + SearchHits.empty(), exprValueFactory, includes, isCountAggRequest()); } else { + // get the value before set searchDone = true + boolean isCountAggRequest = isCountAggRequest(); searchDone = true; return new OpenSearchResponse( searchAction.apply( new SearchRequest().indices(indexName.getIndexNames()).source(sourceBuilder)), exprValueFactory, - includes); + includes, + isCountAggRequest); } } else { // Search with PIT instead of scroll API @@ -181,7 +193,9 @@ public OpenSearchResponse search( public OpenSearchResponse searchWithPIT(Function searchAction) { OpenSearchResponse openSearchResponse; if (searchDone) { - openSearchResponse = new OpenSearchResponse(SearchHits.empty(), exprValueFactory, includes); + openSearchResponse = + new OpenSearchResponse( + SearchHits.empty(), exprValueFactory, includes, isCountAggRequest()); } else { this.sourceBuilder.pointInTimeBuilder(new PointInTimeBuilder(this.pitId)); this.sourceBuilder.timeout(cursorKeepAlive); @@ -200,7 +214,9 @@ public OpenSearchResponse searchWithPIT(Function new SearchRequest().indices(indexName.getIndexNames()).source(this.sourceBuilder); this.searchResponse = searchAction.apply(searchRequest); - openSearchResponse = new OpenSearchResponse(this.searchResponse, exprValueFactory, includes); + openSearchResponse = + new OpenSearchResponse( + this.searchResponse, exprValueFactory, includes, isCountAggRequest()); needClean = openSearchResponse.isEmpty(); searchDone = openSearchResponse.isEmpty(); @@ -226,6 +242,18 @@ public void clean(Consumer cleanAction) { } } + @Override + public void forceClean(Consumer cleanAction) { + try { + if (this.pitId != null) { + cleanAction.accept(this.pitId); + searchDone = true; + } + } finally { + this.pitId = null; + } + } + @Override public boolean hasAnotherBatch() { if (this.pitId != null) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequest.java index f775d552962..93d07a83843 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequest.java @@ -43,6 +43,9 @@ OpenSearchResponse search( */ void clean(Consumer cleanAction); + /** Force clean the request. */ + void forceClean(Consumer cleanAction); + /** * Get the OpenSearchExprValueFactory. * diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java index 3f8a00e4d74..5ba1500f013 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java @@ -48,6 +48,7 @@ import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; +import org.opensearch.sql.opensearch.response.agg.CountAsTotalHitsParser; import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; /** OpenSearch search request builder. */ @@ -225,6 +226,9 @@ public void pushDownAggregation( if (sourceBuilder.sorts() != null) { sourceBuilder.sorts().clear(); } + if (aggregationBuilder.getRight() instanceof CountAsTotalHitsParser) { + sourceBuilder.trackTotalHits(true); + } } /** diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java index d793b53fcac..be5c1f0baf9 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java @@ -102,7 +102,7 @@ public OpenSearchResponse search( openSearchResponse = searchAction.apply(initialSearchRequest); } - var response = new OpenSearchResponse(openSearchResponse, exprValueFactory, includes); + var response = new OpenSearchResponse(openSearchResponse, exprValueFactory, includes, false); needClean = response.isEmpty(); if (!needClean) { setScrollId(openSearchResponse.getScrollId()); @@ -123,6 +123,12 @@ public void clean(Consumer cleanAction) { } } + @Override + public void forceClean(Consumer cleanAction) { + throw new UnsupportedOperationException( + "Force clean is unsupported in OpenSearchScrollRequest"); + } + /** * Is scroll started which means pages after first is being requested. * diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java index a0471c191ed..16380fc6760 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java @@ -48,6 +48,8 @@ public class OpenSearchResponse implements Iterable { /** List of requested include fields. */ private final List includes; + private final boolean isCountAgg; + /** OpenSearchExprValueFactory used to build ExprValue from search result. */ @EqualsAndHashCode.Exclude private final OpenSearchExprValueFactory exprValueFactory; @@ -56,19 +58,33 @@ public OpenSearchResponse( SearchResponse searchResponse, OpenSearchExprValueFactory exprValueFactory, List includes) { + this(searchResponse, exprValueFactory, includes, false); + } + + /** Constructor of OpenSearchResponse. */ + public OpenSearchResponse( + SearchResponse searchResponse, + OpenSearchExprValueFactory exprValueFactory, + List includes, + boolean isCountAgg) { this.hits = searchResponse.getHits(); this.aggregations = searchResponse.getAggregations(); this.exprValueFactory = exprValueFactory; this.includes = includes; + this.isCountAgg = isCountAgg; } /** Constructor of OpenSearchResponse with SearchHits. */ public OpenSearchResponse( - SearchHits hits, OpenSearchExprValueFactory exprValueFactory, List includes) { + SearchHits hits, + OpenSearchExprValueFactory exprValueFactory, + List includes, + boolean isCountAgg) { this.hits = hits; this.aggregations = null; this.exprValueFactory = exprValueFactory; this.includes = includes; + this.isCountAgg = isCountAgg; } /** @@ -78,20 +94,31 @@ public OpenSearchResponse( * @return true for empty */ public boolean isEmpty() { - return (hits.getHits() == null) || (hits.getHits().length == 0) && aggregations == null; + return (hits.getHits() == null) + || (((hits.getHits().length == 0) && aggregations == null) + && (!isCountAgg + || hits.getTotalHits() == null)); // check total hits if is count aggregation } public boolean isAggregationResponse() { return aggregations != null; } + public boolean isCountResponse() { + return isCountAgg; + } + + public int getHitsSize() { + return hits.getHits() == null ? 0 : hits.getHits().length; + } + /** * Make response iterable without need to return internal data structure explicitly. * * @return search hit iterator */ public Iterator iterator() { - if (isAggregationResponse()) { + if (isAggregationResponse() || isCountAgg) { return handleAggregationResponse(); } else { return Arrays.stream(hits.getHits()) @@ -191,7 +218,11 @@ private void addMetaDataFieldsToBuilder( * @return Parsed and built return values from response. */ private Iterator handleAggregationResponse() { - return exprValueFactory.getParser().parse(aggregations).stream() + List> res = + isCountAgg + ? exprValueFactory.getParser().parse(hits) + : exprValueFactory.getParser().parse(aggregations); + return res.stream() .map( entry -> { ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/BucketAggregationParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/BucketAggregationParser.java index 833d0246304..6d63fc01d58 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/BucketAggregationParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/BucketAggregationParser.java @@ -11,6 +11,7 @@ import java.util.Map; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; +import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.Aggregations; import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; @@ -44,4 +45,10 @@ private Map parse(MultiBucketsAggregation.Bucket bucket, String resultMap.putAll(metricsParser.parse(bucket.getAggregations())); return resultMap; } + + @Override + public List> parse(SearchHits hits) { + throw new UnsupportedOperationException( + "BucketAggregationParser doesn't support parse(SearchHits)"); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CompositeAggregationParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CompositeAggregationParser.java index 3f36ffa0f8e..f84aadae676 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CompositeAggregationParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CompositeAggregationParser.java @@ -20,6 +20,7 @@ import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import lombok.Getter; +import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.Aggregations; import org.opensearch.search.aggregations.bucket.composite.CompositeAggregation; @@ -33,6 +34,8 @@ public class CompositeAggregationParser implements OpenSearchAggregationResponseParser { private final MetricParserHelper metricsParser; + // countAggNameList dedicated the list of count aggregations which are filled by doc_count + private List countAggNameList = List.of(); public CompositeAggregationParser(MetricParser... metricParserList) { metricsParser = new MetricParserHelper(Arrays.asList(metricParserList)); @@ -42,6 +45,13 @@ public CompositeAggregationParser(List metricParserList) { metricsParser = new MetricParserHelper(metricParserList); } + /** CompositeAggregationParser with count aggregation name list, used in v3 */ + public CompositeAggregationParser( + List metricParserList, List countAggNameList) { + metricsParser = new MetricParserHelper(metricParserList); + this.countAggNameList = countAggNameList; + } + @Override public List> parse(Aggregations aggregations) { return ((CompositeAggregation) aggregations.asList().get(0)) @@ -52,6 +62,13 @@ private Map parse(CompositeAggregation.Bucket bucket) { Map resultMap = new HashMap<>(); resultMap.putAll(bucket.getKey()); resultMap.putAll(metricsParser.parse(bucket.getAggregations())); + countAggNameList.forEach(name -> resultMap.put(name, bucket.getDocCount())); return resultMap; } + + @Override + public List> parse(SearchHits hits) { + throw new UnsupportedOperationException( + "CompositeAggregationParser doesn't support parse(SearchHits)"); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CountAsTotalHitsParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CountAsTotalHitsParser.java new file mode 100644 index 00000000000..27b4d02c0e6 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CountAsTotalHitsParser.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.response.agg; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.opensearch.search.SearchHits; +import org.opensearch.search.aggregations.Aggregations; + +@Getter +@EqualsAndHashCode +public class CountAsTotalHitsParser implements OpenSearchAggregationResponseParser { + + // countAggNameList dedicated the list of count aggregations which are filled by hits.total.value + private final List countAggNameList; + + public CountAsTotalHitsParser(List countAggNameList) { + this.countAggNameList = countAggNameList; + } + + @Override + public List> parse(Aggregations aggregations) { + throw new UnsupportedOperationException( + "CountAsTotalHitsParser doesn't support parse(Aggregations)"); + } + + @Override + public List> parse(SearchHits hits) { + Map resultMap = new HashMap<>(); + countAggNameList.forEach(name -> resultMap.put(name, hits.getTotalHits().value)); + return Collections.singletonList(resultMap); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/MetricParserHelper.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/MetricParserHelper.java index c2b16863538..58a0edbc611 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/MetricParserHelper.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/MetricParserHelper.java @@ -31,10 +31,20 @@ public class MetricParserHelper { private final Map metricParserMap; + // countAggNameList dedicated the list of count aggregations which are filled by doc_count + private final List countAggNameList; public MetricParserHelper(List metricParserList) { metricParserMap = metricParserList.stream().collect(Collectors.toMap(MetricParser::getName, m -> m)); + this.countAggNameList = List.of(); + } + + /** MetricParserHelper with count aggregation name list, used in v3 */ + public MetricParserHelper(List metricParserList, List countAggNameList) { + metricParserMap = + metricParserList.stream().collect(Collectors.toMap(MetricParser::getName, m -> m)); + this.countAggNameList = countAggNameList; } /** @@ -54,6 +64,7 @@ public Map parse(Aggregations aggregations) { "couldn't parse field %s in aggregation " + "response", aggregation.getName())); } } + // countAggNameList.forEach(name -> resultMap.put(name, bucket.getDocCount())); return resultMap; } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/NoBucketAggregationParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/NoBucketAggregationParser.java index c0a45cba5dc..882a495db2a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/NoBucketAggregationParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/NoBucketAggregationParser.java @@ -18,6 +18,7 @@ import java.util.List; import java.util.Map; import lombok.Getter; +import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.Aggregations; /** No Bucket Aggregation Parser which include only metric parsers. */ @@ -38,4 +39,10 @@ public NoBucketAggregationParser(List metricParserList) { public List> parse(Aggregations aggregations) { return Collections.singletonList(metricsParser.parse(aggregations)); } + + @Override + public List> parse(SearchHits hits) { + throw new UnsupportedOperationException( + "NoBucketAggregationParser doesn't support parse(SearchHits)"); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/OpenSearchAggregationResponseParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/OpenSearchAggregationResponseParser.java index 0c15d72eb60..6990aa6fe8a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/OpenSearchAggregationResponseParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/OpenSearchAggregationResponseParser.java @@ -15,6 +15,7 @@ import java.util.List; import java.util.Map; +import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.Aggregations; /** OpenSearch Aggregation Response Parser. */ @@ -27,4 +28,6 @@ public interface OpenSearchAggregationResponseParser { * @return aggregation result. */ List> parse(Aggregations aggregations); + + List> parse(SearchHits hit); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java index 3895956fef4..85457aafff5 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java @@ -464,6 +464,8 @@ public void apply(OpenSearchRequestBuilder requestBuilder) { public void pushDownSortIntoAggBucket( List collations, List fieldNames) { + // aggregationBuilder.getLeft() could be empty when count agg optimization works + if (aggregationBuilder.getLeft().isEmpty()) return; AggregationBuilder builder = aggregationBuilder.getLeft().get(0); List selected = new ArrayList<>(collations.size()); if (builder instanceof CompositeAggregationBuilder) { @@ -539,6 +541,8 @@ public void pushDownSortIntoAggBucket( * than bucket number. */ public boolean pushDownLimitIntoBucketSize(Integer size) { + // aggregationBuilder.getLeft() could be empty when count agg optimization works + if (aggregationBuilder.getLeft().isEmpty()) return false; AggregationBuilder builder = aggregationBuilder.getLeft().get(0); if (builder instanceof CompositeAggregationBuilder) { CompositeAggregationBuilder compositeAggBuilder = (CompositeAggregationBuilder) builder; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java index 1d9161d6153..77960c734e8 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java @@ -114,6 +114,7 @@ public Enumerator enumerator() { osIndex.getClient(), getFieldPath(), requestBuilder.getMaxResponseSize(), + requestBuilder.getMaxResultWindow(), osIndex.buildRequest(requestBuilder), osIndex.createOpenSearchResourceMonitor()); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexEnumerator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexEnumerator.java index 83c56b61c63..66796f3e4ce 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexEnumerator.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexEnumerator.java @@ -34,11 +34,14 @@ public class OpenSearchIndexEnumerator implements Enumerator { private final List fields; /** Search request. */ - @EqualsAndHashCode.Include @ToString.Include private final OpenSearchRequest request; + @EqualsAndHashCode.Include @ToString.Include private OpenSearchRequest request; /** Largest number of rows allowed in the response. */ @EqualsAndHashCode.Include @ToString.Include private final int maxResponseSize; + /** Largest number of rows allowed in the response. */ + @EqualsAndHashCode.Include @ToString.Include private final int maxResultWindow; + /** How many moveNext() calls to perform resource check once. */ private static final long NUMBER_OF_NEXT_CALL_TO_CHECK = 1000; @@ -53,16 +56,21 @@ public class OpenSearchIndexEnumerator implements Enumerator { private ExprValue current; + /** flag to indicate whether fetch more than one batch */ + private boolean fetchOnce = false; + public OpenSearchIndexEnumerator( OpenSearchClient client, List fields, int maxResponseSize, + int maxResultWindow, OpenSearchRequest request, ResourceMonitor monitor) { this.client = client; this.fields = fields; this.request = request; this.maxResponseSize = maxResponseSize; + this.maxResultWindow = maxResultWindow; this.monitor = monitor; this.queryCount = 0; this.current = null; @@ -73,6 +81,13 @@ public OpenSearchIndexEnumerator( private void fetchNextBatch() { OpenSearchResponse response = client.search(request); + if (response.isAggregationResponse() + || response.isCountResponse() + || response.getHitsSize() < maxResultWindow) { + // no need to fetch next batch if it's for an aggregation + // or the length of response hits is less than max result window size. + fetchOnce = true; + } if (!response.isEmpty()) { iterator = response.iterator(); } else if (iterator == null) { @@ -106,7 +121,7 @@ public boolean moveNext() { throw new NonFallbackCalciteException("insufficient resources to load next row, quit."); } - if (iterator == null || !iterator.hasNext()) { + if (iterator == null || (!iterator.hasNext() && !fetchOnce)) { fetchNextBatch(); } if (iterator.hasNext()) { @@ -132,7 +147,9 @@ public void reset() { @Override public void close() { iterator = Collections.emptyIterator(); - queryCount = 0; - client.cleanup(request); + if (request != null) { + client.forceCleanup(request); + request = null; + } } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java index e304f504242..b3a1d766d8b 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java @@ -278,8 +278,7 @@ void analyze_groupBy() throws ExpressionNotAnalyzableException { assertEquals( "[{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[" + "{\"a\":{\"terms\":{\"field\":\"a\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}," - + "{\"b\":{\"terms\":{\"field\":\"b.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}," - + "\"aggregations\":{\"cnt\":{\"value_count\":{\"field\":\"_index\"}}}}}]", + + "{\"b\":{\"terms\":{\"field\":\"b.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}]", result.getLeft().toString()); assertInstanceOf(CompositeAggregationParser.class, result.getRight()); MetricParserHelper metricsParser = diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java index 54b61e84b15..c84f5098d4b 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java @@ -288,7 +288,7 @@ void response_isnot_aggregation_when_aggregation_is_empty() { void aggregation_iterator() { final List includes = List.of("id1", "id2"); - when(parser.parse(any())) + when(parser.parse((Aggregations) any())) .thenReturn(Arrays.asList(ImmutableMap.of("id1", 1), ImmutableMap.of("id2", 2))); when(searchResponse.getAggregations()).thenReturn(aggregations); when(factory.getParser()).thenReturn(parser); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java index 2ad71f9dc6a..b5d154273cc 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java @@ -58,6 +58,119 @@ public void testSimpleCount() { verifyPPLToSparkSQL(root, expectedSparkSql); } + @Test + public void testCountField() { + String ppl = "source=EMP | stats count(COMM) as c"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "" + + "LogicalAggregate(group=[{}], c=[COUNT($0)])\n" + + " LogicalProject(COMM=[$6])\n" + + " LogicalFilter(condition=[IS NOT NULL($6)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "c=4\n"; + verifyResult(root, expectedResult); + String expectedSparkSql = + "SELECT COUNT(`COMM`) `c`\nFROM `scott`.`EMP`\nWHERE `COMM` IS NOT NULL"; + verifyPPLToSparkSQL(root, expectedSparkSql); + + ppl = "source=EMP | stats count(COMM) as c1, count(COMM) as c2"; + root = getRelNode(ppl); + expectedLogical = + "" + + "LogicalProject(c1=[$0], c2=[$0])\n" + + " LogicalAggregate(group=[{}], c1=[COUNT($0)])\n" + + " LogicalProject(COMM=[$6])\n" + + " LogicalFilter(condition=[IS NOT NULL($6)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + expectedResult = "c1=4; c2=4\n"; + verifyResult(root, expectedResult); + expectedSparkSql = + "SELECT COUNT(`COMM`) `c1`, COUNT(`COMM`) `c2`\n" + + "FROM `scott`.`EMP`\n" + + "WHERE `COMM` IS NOT NULL"; + verifyPPLToSparkSQL(root, expectedSparkSql); + + ppl = "source=EMP | stats count(), count(COMM) as c"; + root = getRelNode(ppl); + expectedLogical = + "" + + "LogicalAggregate(group=[{}], count()=[COUNT()], c=[COUNT($0)])\n" + + " LogicalProject(COMM=[$6])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + expectedResult = "count()=14; c=4\n"; + verifyResult(root, expectedResult); + expectedSparkSql = "SELECT COUNT(*) `count()`, COUNT(`COMM`) `c`\nFROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + + ppl = "source=EMP | stats count(COMM + 1.0) as c"; + root = getRelNode(ppl); + expectedLogical = + "" + + "LogicalAggregate(group=[{}], c=[COUNT($0)])\n" + + " LogicalProject($f1=[+($6, 1.0:DECIMAL(2, 1))])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + expectedResult = "c=4\n"; + verifyResult(root, expectedResult); + + expectedSparkSql = "SELECT COUNT(`COMM` + 1.0) `c`\nFROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + + ppl = "source=EMP | stats count(eval(COMM >= 500)) as c"; + root = getRelNode(ppl); + expectedLogical = + "" + + "LogicalAggregate(group=[{}], c=[COUNT($0)])\n" + + " LogicalProject($f1=[CASE(>=($6, 500), 1, null:NULL)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + expectedResult = "c=2\n"; + verifyResult(root, expectedResult); + + expectedSparkSql = + "SELECT COUNT(CASE WHEN `COMM` >= 500 THEN 1 ELSE NULL END) `c`\nFROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + + ppl = "source=EMP | eval COMM1 = COMM | stats count(COMM) as c, count(COMM1) as c1"; + root = getRelNode(ppl); + expectedLogical = + "LogicalAggregate(group=[{}], c=[COUNT($0)], c1=[COUNT($1)])\n" + + " LogicalProject(COMM=[$6], COMM1=[$8])\n" + + " LogicalFilter(condition=[IS NOT NULL($6)])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], COMM1=[$6])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + expectedResult = "c=4; c1=4\n"; + verifyResult(root, expectedResult); + + expectedSparkSql = + "SELECT COUNT(`COMM`) `c`, COUNT(`COMM1`) `c1`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " `COMM` `COMM1`\n" + + "FROM `scott`.`EMP`) `t12`\n" + + "WHERE `COMM` IS NOT NULL"; + verifyPPLToSparkSQL(root, expectedSparkSql); + + ppl = "source=EMP | eval COMM1 = COMM + 1 | stats count(COMM) as c, count(COMM1) as c1"; + root = getRelNode(ppl); + expectedLogical = + "" + + "LogicalAggregate(group=[{}], c=[COUNT($0)], c1=[COUNT($1)])\n" + + " LogicalProject(COMM=[$6], COMM1=[+($6, 1)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + expectedResult = "c=4; c1=4\n"; + verifyResult(root, expectedResult); + + expectedSparkSql = "SELECT COUNT(`COMM`) `c`, COUNT(`COMM` + 1) `c1`\nFROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test public void testTakeAgg() { String ppl = "source=EMP | stats take(JOB, 2) as c"; @@ -384,8 +497,7 @@ public void testCountDistinct() { verifyResult(root, expectedResult); String expectedSparkSql = - "" - + "SELECT COUNT(DISTINCT `JOB`) `distinct_count(JOB)`, `DEPTNO`\n" + "SELECT COUNT(DISTINCT `JOB`) `distinct_count(JOB)`, `DEPTNO`\n" + "FROM `scott`.`EMP`\n" + "GROUP BY `DEPTNO`"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -406,10 +518,7 @@ public void testCountDistinctWithAlias() { verifyResult(root, expectedResult); String expectedSparkSql = - "" - + "SELECT COUNT(DISTINCT `JOB`) `dc`, `DEPTNO`\n" - + "FROM `scott`.`EMP`\n" - + "GROUP BY `DEPTNO`"; + "SELECT COUNT(DISTINCT `JOB`) `dc`, `DEPTNO`\nFROM `scott`.`EMP`\nGROUP BY `DEPTNO`"; verifyPPLToSparkSQL(root, expectedSparkSql); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java index 37c42cb8b33..150121448f4 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java @@ -476,11 +476,11 @@ public void testJoinWithRelationSubquery() { + " | sort - DEPTNO\n" + " | head 10\n" + " ]\n" - + "| stats count(MGR) as cnt by JOB\n"; + + "| stats sum(MGR) as sum by JOB\n"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(cnt=[$1], JOB=[$0])\n" - + " LogicalAggregate(group=[{0}], cnt=[COUNT($1)])\n" + "LogicalProject(sum=[$1], JOB=[$0])\n" + + " LogicalAggregate(group=[{0}], sum=[SUM($1)])\n" + " LogicalProject(JOB=[$2], MGR=[$3])\n" + " LogicalJoin(condition=[=($7, $8)], joinType=[inner])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" @@ -489,12 +489,12 @@ public void testJoinWithRelationSubquery() { + " LogicalFilter(condition=[AND(>($0, 10), =($2, 'CHICAGO':VARCHAR))])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n"; verifyLogical(root, expectedLogical); - String expectedResult = "cnt=4; JOB=SALESMAN\ncnt=1; JOB=CLERK\ncnt=1; JOB=MANAGER\n"; + String expectedResult = "sum=30792; JOB=SALESMAN\nsum=7698; JOB=CLERK\nsum=7839; JOB=MANAGER\n"; verifyResult(root, expectedResult); String expectedSparkSql = "" - + "SELECT COUNT(`EMP`.`MGR`) `cnt`, `EMP`.`JOB`\n" + + "SELECT SUM(`EMP`.`MGR`) `sum`, `EMP`.`JOB`\n" + "FROM `scott`.`EMP`\n" + "INNER JOIN (SELECT `DEPTNO`, `DNAME`\n" + "FROM `scott`.`DEPT`\n"