diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java index f2d44412ca7..f436dfe886c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java @@ -501,22 +501,22 @@ public void testStatsBySpanNonBucketNullable() throws IOException { @Test public void testStatsByTimeSpan() throws IOException { - String expected = loadExpectedPlan("explain_stats_by_timespan.json"); - assertJsonEqualsIgnoreId( + String expected = loadExpectedPlan("explain_stats_by_timespan.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format("source=%s | stats count() by span(birthdate,1m)", TEST_INDEX_BANK))); - expected = loadExpectedPlan("explain_stats_by_timespan2.json"); - assertJsonEqualsIgnoreId( + expected = loadExpectedPlan("explain_stats_by_timespan2.yaml"); + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format("source=%s | stats count() by span(birthdate,1M)", TEST_INDEX_BANK))); // bucket_nullable doesn't impact by-span-time - assertJsonEqualsIgnoreId( + assertYamlEqualsIgnoreId( expected, - explainQueryToString( + explainQueryYaml( String.format( "source=%s | stats bucket_nullable=false count() by span(birthdate,1M)", TEST_INDEX_BANK))); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java index f06e3ba775b..57258748723 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java @@ -59,6 +59,8 @@ protected String executeQueryToString(String query) throws IOException { return getResponseBody(response, true); } + /** Deprecated, use {@link #explainQueryYaml(String)} */ + @Deprecated protected String explainQueryToString(String query) throws IOException { return explainQueryToString(query, false); } diff --git a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java index 05ac95bb19e..8ea3c6f8b87 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java @@ -380,7 +380,11 @@ public static void assertJsonEquals(String expected, String actual) { JsonParser.parseString(eliminatePid(actual))); } - /** Compare two JSON string are equals with ignoring the RelNode id in the Calcite plan. */ + /** + * Compare two JSON string are equals with ignoring the RelNode id in the Calcite plan. + * Deprecated, use {@link #assertYamlEqualsIgnoreId(String, String)} + */ + @Deprecated public static void assertJsonEqualsIgnoreId(String expected, String actual) { assertJsonEquals(cleanUpId(expected), cleanUpId(actual)); } @@ -408,6 +412,7 @@ private static String eliminatePid(String s) { return s.replaceAll("pitId=[^,]+,", "pitId=*,"); } + /** Compare two YAML strings are equals with ignoring the RelNode id in the Calcite plan. */ public static void assertYamlEqualsIgnoreId(String expectedYaml, String actualYaml) { String cleanedYaml = cleanUpYaml(actualYaml); String cleanedExpectedYaml = cleanUpYaml(expectedYaml); diff --git a/integ-test/src/test/resources/big5/queries/composite_date_histogram_daily.ppl b/integ-test/src/test/resources/big5/queries/composite_date_histogram_daily.ppl index 656289b0603..10679eb2ea6 100644 --- a/integ-test/src/test/resources/big5/queries/composite_date_histogram_daily.ppl +++ b/integ-test/src/test/resources/big5/queries/composite_date_histogram_daily.ppl @@ -31,4 +31,5 @@ */ source = big5 | where `@timestamp` >= '2022-12-30 00:00:00' and `@timestamp` < '2023-01-07 12:00:00' -| stats count() by span(`@timestamp`, 1d) \ No newline at end of file +| stats count() by span(`@timestamp`, 1d) +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/composite_terms.ppl b/integ-test/src/test/resources/big5/queries/composite_terms.ppl index 07edca09e69..4a9a4a3244c 100644 --- a/integ-test/src/test/resources/big5/queries/composite_terms.ppl +++ b/integ-test/src/test/resources/big5/queries/composite_terms.ppl @@ -29,4 +29,5 @@ source = big5 | where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-02 10:00:00' | stats count() by `process.name`, `cloud.region` -| sort - `process.name`, + `cloud.region` \ No newline at end of file +| sort - `process.name`, + `cloud.region` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/composite_terms_keyword.ppl b/integ-test/src/test/resources/big5/queries/composite_terms_keyword.ppl index 42b8c9585a4..e0a92c0b6e6 100644 --- a/integ-test/src/test/resources/big5/queries/composite_terms_keyword.ppl +++ b/integ-test/src/test/resources/big5/queries/composite_terms_keyword.ppl @@ -30,4 +30,5 @@ source = big5 | where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-02 10:00:00' | stats count() by `process.name`, `cloud.region`, `aws.cloudwatch.log_stream` -| sort - `process.name`, + `cloud.region`, + `aws.cloudwatch.log_stream` \ No newline at end of file +| sort - `process.name`, + `cloud.region`, + `aws.cloudwatch.log_stream` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/composite_terms.ppl b/integ-test/src/test/resources/big5/queries/optimized/composite_terms.ppl index 97897e227de..6161a31a0c8 100644 --- a/integ-test/src/test/resources/big5/queries/optimized/composite_terms.ppl +++ b/integ-test/src/test/resources/big5/queries/optimized/composite_terms.ppl @@ -29,4 +29,5 @@ source = big5 | where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-02 10:00:00' | stats bucket_nullable = false count() by `process.name`, `cloud.region` -| sort - `process.name`, + `cloud.region` \ No newline at end of file +| sort - `process.name`, + `cloud.region` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/optimized/composite_terms_keyword.ppl b/integ-test/src/test/resources/big5/queries/optimized/composite_terms_keyword.ppl index 04d12b4fb0e..224f5575aaa 100644 --- a/integ-test/src/test/resources/big5/queries/optimized/composite_terms_keyword.ppl +++ b/integ-test/src/test/resources/big5/queries/optimized/composite_terms_keyword.ppl @@ -30,4 +30,5 @@ source = big5 | where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-02 10:00:00' | stats bucket_nullable = false count() by `process.name`, `cloud.region`, `aws.cloudwatch.log_stream` -| sort - `process.name`, + `cloud.region`, + `aws.cloudwatch.log_stream` \ No newline at end of file +| sort - `process.name`, + `cloud.region`, + `aws.cloudwatch.log_stream` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_date_histogram_daily.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_date_histogram_daily.yaml index 9b69c67b74c..10023133a38 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_date_histogram_daily.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_date_histogram_daily.yaml @@ -1,11 +1,12 @@ calcite: logical: | LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalProject(count()=[$1], span(`@timestamp`,1d)=[$0]) - LogicalAggregate(group=[{0}], count()=[COUNT()]) - LogicalProject(span(`@timestamp`,1d)=[SPAN($17, 1, 'd')]) - LogicalFilter(condition=[IS NOT NULL($17)]) - LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2022-12-30 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-07 12:00:00':VARCHAR)))]) - CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) + LogicalSort(fetch=[10]) + LogicalProject(count()=[$1], span(`@timestamp`,1d)=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(span(`@timestamp`,1d)=[SPAN($17, 1, 'd')]) + LogicalFilter(condition=[IS NOT NULL($17)]) + LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2022-12-30 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-07 12:00:00':VARCHAR)))]) + CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[@timestamp], FILTER->SEARCH($0, Sarg[['2022-12-30 00:00:00':VARCHAR..'2023-01-07 12:00:00':VARCHAR); NULL AS FALSE]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1d)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2022-12-30T00:00:00.000Z","to":"2023-01-07T12:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"exists":{"field":"@timestamp","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["@timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"span(`@timestamp`,1d)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[@timestamp], FILTER->SEARCH($0, Sarg[['2022-12-30 00:00:00':VARCHAR..'2023-01-07 12:00:00':VARCHAR); NULL AS FALSE]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1d)], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2022-12-30T00:00:00.000Z","to":"2023-01-07T12:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["@timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"span(`@timestamp`,1d)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms.yaml index 8720f023f80..cc3af323ddf 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms.yaml @@ -1,7 +1,7 @@ calcite: logical: | LogicalSystemLimit(sort0=[$1], sort1=[$2], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalSort(sort0=[$1], sort1=[$2], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first]) + LogicalSort(sort0=[$1], sort1=[$2], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first], fetch=[10]) LogicalProject(count()=[$2], process.name=[$0], cloud.region=[$1]) LogicalAggregate(group=[{0, 1}], count()=[COUNT()]) LogicalProject(process.name=[$7], cloud.region=[$14]) @@ -9,4 +9,4 @@ calcite: LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-02 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-02 10:00:00':VARCHAR)))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, cloud.region, @timestamp], FILTER->SEARCH($2, Sarg[['2023-01-02 00:00:00':VARCHAR..'2023-01-02 10:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), process.name, cloud.region], SORT->[1 DESC LAST, 2 ASC FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-02T00:00:00.000Z","to":"2023-01-02T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["process.name","cloud.region","@timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":false,"order":"desc"}}},{"cloud.region":{"terms":{"field":"cloud.region","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, cloud.region, @timestamp], FILTER->SEARCH($2, Sarg[['2023-01-02 00:00:00':VARCHAR..'2023-01-02 10:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), process.name, cloud.region], SORT->[1 DESC LAST, 2 ASC FIRST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-02T00:00:00.000Z","to":"2023-01-02T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["process.name","cloud.region","@timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":false,"order":"desc"}}},{"cloud.region":{"terms":{"field":"cloud.region","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms_keyword.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms_keyword.yaml index ac251d900f0..9e546a26dbf 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms_keyword.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms_keyword.yaml @@ -1,7 +1,7 @@ calcite: logical: | LogicalSystemLimit(sort0=[$1], sort1=[$2], sort2=[$3], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first], dir2=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) - LogicalSort(sort0=[$1], sort1=[$2], sort2=[$3], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first], dir2=[ASC-nulls-first]) + LogicalSort(sort0=[$1], sort1=[$2], sort2=[$3], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first], dir2=[ASC-nulls-first], fetch=[10]) LogicalProject(count()=[$3], process.name=[$0], cloud.region=[$1], aws.cloudwatch.log_stream=[$2]) LogicalAggregate(group=[{0, 1, 2}], count()=[COUNT()]) LogicalProject(process.name=[$7], cloud.region=[$14], aws.cloudwatch.log_stream=[$34]) @@ -9,4 +9,4 @@ calcite: LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-02 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-02 10:00:00':VARCHAR)))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, cloud.region, @timestamp, aws.cloudwatch.log_stream], FILTER->SEARCH($2, Sarg[['2023-01-02 00:00:00':VARCHAR..'2023-01-02 10:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},count()=COUNT()), PROJECT->[count(), process.name, cloud.region, aws.cloudwatch.log_stream], SORT->[1 DESC LAST, 2 ASC FIRST, 3 ASC FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-02T00:00:00.000Z","to":"2023-01-02T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["process.name","cloud.region","@timestamp","aws.cloudwatch.log_stream"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":false,"order":"desc"}}},{"cloud.region":{"terms":{"field":"cloud.region","missing_bucket":false,"order":"asc"}}},{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, cloud.region, @timestamp, aws.cloudwatch.log_stream], FILTER->SEARCH($2, Sarg[['2023-01-02 00:00:00':VARCHAR..'2023-01-02 10:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},count()=COUNT()), PROJECT->[count(), process.name, cloud.region, aws.cloudwatch.log_stream], SORT->[1 DESC LAST, 2 ASC FIRST, 3 ASC FIRST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-02T00:00:00.000Z","to":"2023-01-02T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["process.name","cloud.region","@timestamp","aws.cloudwatch.log_stream"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":false,"order":"desc"}}},{"cloud.region":{"terms":{"field":"cloud.region","missing_bucket":false,"order":"asc"}}},{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/date_histogram_minute_agg.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/date_histogram_minute_agg.yaml index c715c2c2a42..44b15522967 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/date_histogram_minute_agg.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/date_histogram_minute_agg.yaml @@ -8,4 +8,4 @@ calcite: LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[@timestamp], FILTER->SEARCH($0, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR); NULL AS FALSE]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1m)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"exists":{"field":"@timestamp","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["@timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"span(`@timestamp`,1m)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[@timestamp], FILTER->SEARCH($0, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR); NULL AS FALSE]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1m)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["@timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"span(`@timestamp`,1m)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_with_search.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_with_search.yaml index 29ebac7168f..bd8114a7989 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_with_search.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_with_search.yaml @@ -8,4 +8,4 @@ calcite: LogicalFilter(condition=[AND(>=($3, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($3, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[birthdate], FILTER->SEARCH($0, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR); NULL AS FALSE]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1d)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"birthdate":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"exists":{"field":"birthdate","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["birthdate"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(birthdate,1d)":{"date_histogram":{"field":"birthdate","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[birthdate], FILTER->SEARCH($0, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR); NULL AS FALSE]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1d)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"birthdate":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["birthdate"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(birthdate,1d)":{"date_histogram":{"field":"birthdate","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.json deleted file mode 100644 index f84aa0cb018..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(birthdate,1m)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(birthdate,1m)=[SPAN($3, 1, 'm')])\n LogicalFilter(condition=[IS NOT NULL($3)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1m)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(birthdate,1m)\":{\"date_histogram\":{\"field\":\"birthdate\",\"missing_bucket\":false,\"order\":\"asc\",\"fixed_interval\":\"1m\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.yaml new file mode 100644 index 00000000000..b4384528c0c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], span(birthdate,1m)=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(span(birthdate,1m)=[SPAN($3, 1, 'm')]) + LogicalFilter(condition=[IS NOT NULL($3)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1m)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(birthdate,1m)":{"date_histogram":{"field":"birthdate","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.json deleted file mode 100644 index 036547978b1..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(birthdate,1M)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(birthdate,1M)=[SPAN($3, 1, 'M')])\n LogicalFilter(condition=[IS NOT NULL($3)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1M)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(birthdate,1M)\":{\"date_histogram\":{\"field\":\"birthdate\",\"missing_bucket\":false,\"order\":\"asc\",\"calendar_interval\":\"1M\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.yaml new file mode 100644 index 00000000000..5021adf62b8 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], span(birthdate,1M)=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(span(birthdate,1M)=[SPAN($3, 1, 'M')]) + LogicalFilter(condition=[IS NOT NULL($3)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1M)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(birthdate,1M)":{"date_histogram":{"field":"birthdate","missing_bucket":false,"order":"asc","calendar_interval":"1M"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan.json deleted file mode 100644 index 1b846e6c16e..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(birthdate,1m)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(birthdate,1m)=[SPAN($3, 1, 'm')])\n LogicalFilter(condition=[IS NOT NULL($3)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(birthdate,1m)=[$t0])\n EnumerableAggregate(group=[{0}], count()=[COUNT()])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1], expr#20=['m'], expr#21=[SPAN($t3, $t19, $t20)], expr#22=[IS NOT NULL($t3)], span(birthdate,1m)=[$t21], $condition=[$t22])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan.yaml new file mode 100644 index 00000000000..3843b2bce4a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], span(birthdate,1m)=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(span(birthdate,1m)=[SPAN($3, 1, 'm')]) + LogicalFilter(condition=[IS NOT NULL($3)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(birthdate,1m)=[$t0]) + EnumerableAggregate(group=[{0}], count()=[COUNT()]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1], expr#20=['m'], expr#21=[SPAN($t3, $t19, $t20)], expr#22=[IS NOT NULL($t3)], span(birthdate,1m)=[$t21], $condition=[$t22]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan2.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan2.json deleted file mode 100644 index 9cec9bcf190..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan2.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(birthdate,1M)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(birthdate,1M)=[SPAN($3, 1, 'M')])\n LogicalFilter(condition=[IS NOT NULL($3)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(birthdate,1M)=[$t0])\n EnumerableAggregate(group=[{0}], count()=[COUNT()])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1], expr#20=['M'], expr#21=[SPAN($t3, $t19, $t20)], expr#22=[IS NOT NULL($t3)], span(birthdate,1M)=[$t21], $condition=[$t22])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan2.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan2.yaml new file mode 100644 index 00000000000..af739c44d85 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan2.yaml @@ -0,0 +1,14 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], span(birthdate,1M)=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(span(birthdate,1M)=[SPAN($3, 1, 'M')]) + LogicalFilter(condition=[IS NOT NULL($3)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(birthdate,1M)=[$t0]) + EnumerableAggregate(group=[{0}], count()=[COUNT()]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1], expr#20=['M'], expr#21=[SPAN($t3, $t19, $t20)], expr#22=[IS NOT NULL($t3)], span(birthdate,1M)=[$t21], $condition=[$t22]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_date_histogram_daily.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_date_histogram_daily.yaml index 9a0882dc49a..073078ddf0d 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_date_histogram_daily.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_date_histogram_daily.yaml @@ -3,17 +3,23 @@ root: description: fields: "[count(), span(`@timestamp`,1d)]" children: - - name: OpenSearchIndexScan + - name: LimitOperator description: - request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ - size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\"\ - :{\"@timestamp\":{\"from\":1672358400000,\"to\":null,\"include_lower\":true,\"\ - include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\":{\"from\"\ - :null,\"to\":1673092800000,\"include_lower\":true,\"include_upper\":false,\"\ - boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\"\ - :{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(`@timestamp`,1d)\"\ - :{\"date_histogram\":{\"field\":\"@timestamp\",\"missing_bucket\":false,\"\ - order\":\"asc\",\"fixed_interval\":\"1d\"}}}]},\"aggregations\":{\"count()\"\ - :{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false,\ - \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - children: [] \ No newline at end of file + limit: 10 + offset: 0 + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"\ + range\":{\"@timestamp\":{\"from\":1672358400000,\"to\":null,\"include_lower\"\ + :true,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\"\ + :{\"from\":null,\"to\":1673092800000,\"include_lower\":true,\"include_upper\"\ + :false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"\ + aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"\ + sources\":[{\"span(`@timestamp`,1d)\":{\"date_histogram\":{\"field\"\ + :\"@timestamp\",\"missing_bucket\":false,\"order\":\"asc\",\"fixed_interval\"\ + :\"1d\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\"\ + :\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_terms.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_terms.yaml index 481d9cdd423..6a0e0c660da 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_terms.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_terms.yaml @@ -3,19 +3,25 @@ root: description: fields: "[count(), process.name, cloud.region]" children: - - name: OpenSearchIndexScan + - name: LimitOperator description: - request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ - size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\"\ - :{\"@timestamp\":{\"from\":1672617600000,\"to\":null,\"include_lower\":true,\"\ - include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\":{\"from\"\ - :null,\"to\":1672653600000,\"include_lower\":true,\"include_upper\":false,\"\ - boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\"\ - :{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"process.name\"\ - :{\"terms\":{\"field\":\"process.name\",\"missing_bucket\":true,\"missing_order\"\ - :\"last\",\"order\":\"desc\"}}},{\"cloud.region\":{\"terms\":{\"field\"\ - :\"cloud.region\",\"missing_bucket\":true,\"missing_order\":\"first\",\"\ - order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"\ - field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - children: [] \ No newline at end of file + limit: 10 + offset: 0 + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"\ + range\":{\"@timestamp\":{\"from\":1672617600000,\"to\":null,\"include_lower\"\ + :true,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\"\ + :{\"from\":null,\"to\":1672653600000,\"include_lower\":true,\"include_upper\"\ + :false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"\ + aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"\ + sources\":[{\"process.name\":{\"terms\":{\"field\":\"process.name\"\ + ,\"missing_bucket\":true,\"missing_order\":\"last\",\"order\":\"desc\"\ + }}},{\"cloud.region\":{\"terms\":{\"field\":\"cloud.region\",\"missing_bucket\"\ + :true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\"\ + :{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_terms_keyword.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_terms_keyword.yaml index a7f12407647..5c77f33d0cd 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_terms_keyword.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_terms_keyword.yaml @@ -3,21 +3,27 @@ root: description: fields: "[count(), process.name, cloud.region, aws.cloudwatch.log_stream]" children: - - name: OpenSearchIndexScan + - name: LimitOperator description: - request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ - size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\"\ - :{\"@timestamp\":{\"from\":1672617600000,\"to\":null,\"include_lower\":true,\"\ - include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\":{\"from\"\ - :null,\"to\":1672653600000,\"include_lower\":true,\"include_upper\":false,\"\ - boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"aggregations\"\ - :{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"process.name\"\ - :{\"terms\":{\"field\":\"process.name\",\"missing_bucket\":true,\"missing_order\"\ - :\"last\",\"order\":\"desc\"}}},{\"cloud.region\":{\"terms\":{\"field\"\ - :\"cloud.region\",\"missing_bucket\":true,\"missing_order\":\"first\",\"\ - order\":\"asc\"}}},{\"aws.cloudwatch.log_stream\":{\"terms\":{\"field\"\ - :\"aws.cloudwatch.log_stream\",\"missing_bucket\":true,\"missing_order\"\ - :\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\"\ - :{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - children: [] \ No newline at end of file + limit: 10 + offset: 0 + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\"\ + :0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"\ + range\":{\"@timestamp\":{\"from\":1672617600000,\"to\":null,\"include_lower\"\ + :true,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"@timestamp\"\ + :{\"from\":null,\"to\":1672653600000,\"include_lower\":true,\"include_upper\"\ + :false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"\ + aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"\ + sources\":[{\"process.name\":{\"terms\":{\"field\":\"process.name\"\ + ,\"missing_bucket\":true,\"missing_order\":\"last\",\"order\":\"desc\"\ + }}},{\"cloud.region\":{\"terms\":{\"field\":\"cloud.region\",\"missing_bucket\"\ + :true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"aws.cloudwatch.log_stream\"\ + :{\"terms\":{\"field\":\"aws.cloudwatch.log_stream\",\"missing_bucket\"\ + :true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\"\ + :{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true,\ + \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan.json deleted file mode 100644 index 7d345202ce4..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[count(), span(birthdate,1m)]" - }, - "children": [{ - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_bank, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(birthdate,1m)\":{\"date_histogram\":{\"field\":\"birthdate\",\"missing_bucket\":false,\"order\":\"asc\",\"fixed_interval\":\"1m\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - }] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan.yaml new file mode 100644 index 00000000000..3fd26cb19a6 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan.yaml @@ -0,0 +1,15 @@ +root: + name: ProjectOperator + description: + fields: "[count(), span(birthdate,1m)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_bank,\ + \ sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\"\ + :{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(birthdate,1m)\"\ + :{\"date_histogram\":{\"field\":\"birthdate\",\"missing_bucket\":false,\"\ + order\":\"asc\",\"fixed_interval\":\"1m\"}}}]},\"aggregations\":{\"count()\"\ + :{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan2.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan2.json deleted file mode 100644 index a5b9e210f09..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan2.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[count(), span(birthdate,1M)]" - }, - "children": [{ - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_bank, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(birthdate,1M)\":{\"date_histogram\":{\"field\":\"birthdate\",\"missing_bucket\":false,\"order\":\"asc\",\"calendar_interval\":\"1M\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - }] - } -} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan2.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan2.yaml new file mode 100644 index 00000000000..167328625f1 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan2.yaml @@ -0,0 +1,15 @@ +root: + name: ProjectOperator + description: + fields: "[count(), span(birthdate,1M)]" + children: + - name: OpenSearchIndexScan + description: + request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_bank,\ + \ sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\"\ + :{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(birthdate,1M)\"\ + :{\"date_histogram\":{\"field\":\"birthdate\",\"missing_bucket\":false,\"\ + order\":\"asc\",\"calendar_interval\":\"1M\"}}}]},\"aggregations\":{\"count()\"\ + :{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false,\ + \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + children: [] \ No newline at end of file diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index 209e7acbb32..b31fe86479e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -667,9 +667,9 @@ private QueryExpression binary(RexCall call) { switch (nullAs) { case FALSE: // e.g. where isNotNull(a) and (a = 1 or a = 2) - // TODO: For this case, seems return `expression` should be equivalent - finalExpression = CompoundQueryExpression.and( - false, expression, QueryExpression.create(pair.getKey()).exists()); + // For this case, return `expression` is equivalent + // But DSL `bool.must` could slow down the query, so we return `expression` + finalExpression = expression; break; case TRUE: // e.g. where isNull(a) or a = 1 or a = 2 diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java index 27dac8c21f6..e69de29bb2d 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilder.java @@ -1,133 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.script.aggregation.dsl; - -import static org.opensearch.sql.data.type.ExprCoreType.DATE; -import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; -import static org.opensearch.sql.data.type.ExprCoreType.TIME; -import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; -import static org.opensearch.sql.opensearch.storage.script.aggregation.AggregationQueryBuilder.AGGREGATION_BUCKET_SIZE; - -import java.util.List; -import java.util.stream.Collectors; -import org.opensearch.search.aggregations.BucketOrder; -import org.opensearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; -import org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; -import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.opensearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder; -import org.opensearch.search.aggregations.bucket.terms.MultiTermsAggregationBuilder; -import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.opensearch.search.aggregations.support.MultiTermsValuesSourceConfig; -import org.opensearch.search.aggregations.support.ValueType; -import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; -import org.opensearch.sql.ast.expression.SpanUnit; -import org.opensearch.sql.expression.NamedExpression; -import org.opensearch.sql.expression.span.SpanExpression; -import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; -import org.opensearch.sql.opensearch.storage.serde.ExpressionSerializer; - -/** Bucket Aggregation Builder. */ -public class BucketAggregationBuilder { - - private final AggregationBuilderHelper helper; - - public BucketAggregationBuilder(ExpressionSerializer serializer) { - this.helper = new AggregationBuilderHelper(serializer); - } - - /** Build the ValuesSourceAggregationBuilder. */ - public ValuesSourceAggregationBuilder build(NamedExpression expr) { - if (expr.getDelegated() instanceof SpanExpression) { - SpanExpression spanExpr = (SpanExpression) expr.getDelegated(); - return buildHistogram( - expr.getName(), - spanExpr.getField().toString(), - spanExpr.getValue().valueOf().doubleValue(), - spanExpr.getUnit()); - } else { - TermsAggregationBuilder sourceBuilder = new TermsAggregationBuilder(expr.getName()); - sourceBuilder.size(AGGREGATION_BUCKET_SIZE); - sourceBuilder.order(BucketOrder.key(true)); - // Time types values are converted to LONG in ExpressionAggregationScript::execute - if ((expr.getDelegated().type() instanceof OpenSearchDateType - && List.of(TIMESTAMP, TIME, DATE, DATETIME) - .contains(((OpenSearchDateType) expr.getDelegated().type()).getExprCoreType())) - || List.of(TIMESTAMP, TIME, DATE, DATETIME).contains(expr.getDelegated().type())) { - sourceBuilder.userValueTypeHint(ValueType.LONG); - } - return helper.build(expr.getDelegated(), sourceBuilder::field, sourceBuilder::script); - } - } - - /** Build the MultiTermsAggregationBuilder. */ - public MultiTermsAggregationBuilder buildMultipleTerms(List exprs) { - MultiTermsAggregationBuilder sourceBuilder = - new MultiTermsAggregationBuilder( - exprs.stream().map(NamedExpression::getName).collect(Collectors.joining("_"))); - sourceBuilder.terms( - exprs.stream() - .map( - expr -> { - MultiTermsValuesSourceConfig.Builder config = - new MultiTermsValuesSourceConfig.Builder(); - config.setFieldName(expr.getName()); - // Time types values are converted to LONG in ExpressionAggregationScript::execute - if ((expr.getDelegated().type() instanceof OpenSearchDateType - && List.of(TIMESTAMP, TIME, DATE) - .contains( - ((OpenSearchDateType) expr.getDelegated().type()) - .getExprCoreType())) - || List.of(TIMESTAMP, TIME, DATE).contains(expr.getDelegated().type())) { - config.setUserValueTypeHint(ValueType.LONG); - } - return config.build(); - }) - .collect(Collectors.toList())); - sourceBuilder.size(AGGREGATION_BUCKET_SIZE); - return sourceBuilder; - } - - public static ValuesSourceAggregationBuilder buildHistogram( - String name, String field, Double value, SpanUnit unit) { - switch (unit) { - case NONE: - return new HistogramAggregationBuilder(name).field(field).interval(value); - case UNKNOWN: - throw new IllegalStateException("Invalid span unit"); - default: - return buildDateHistogram(name, field, value.intValue(), unit); - } - } - - public static ValuesSourceAggregationBuilder buildAutoDateHistogram( - String name, String field, Integer bucketSize) { - return new AutoDateHistogramAggregationBuilder(name).field(field).setNumBuckets(bucketSize); - } - - public static ValuesSourceAggregationBuilder buildDateHistogram( - String name, String field, Integer value, SpanUnit unit) { - String spanValue = value + unit.getName(); - switch (unit) { - case MILLISECOND: - case MS: - case SECOND: - case S: - case MINUTE: - case m: - case HOUR: - case H: - case DAY: - case D: - return new DateHistogramAggregationBuilder(name) - .field(field) - .fixedInterval(new DateHistogramInterval(spanValue)); - default: - return new DateHistogramAggregationBuilder(name) - .field(field) - .calendarInterval(new DateHistogramInterval(spanValue)); - } - } -} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java deleted file mode 100644 index 02ca2a5d312..00000000000 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.script.aggregation.dsl; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.when; -import static org.opensearch.core.xcontent.ToXContent.EMPTY_PARAMS; -import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; -import static org.opensearch.sql.expression.DSL.literal; -import static org.opensearch.sql.expression.DSL.named; -import static org.opensearch.sql.expression.DSL.ref; - -import java.util.Map; -import lombok.SneakyThrows; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayNameGeneration; -import org.junit.jupiter.api.DisplayNameGenerator; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.core.common.bytes.BytesReference; -import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; -import org.opensearch.sql.data.type.ExprCoreType; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.expression.DSL; -import org.opensearch.sql.expression.NamedExpression; -import org.opensearch.sql.expression.parse.ParseExpression; -import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; -import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; -import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; -import org.opensearch.sql.opensearch.storage.serde.ExpressionSerializer; - -@DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) -@ExtendWith(MockitoExtension.class) -class BucketAggregationBuilderTest { - - @Mock private ExpressionSerializer serializer; - - private BucketAggregationBuilder aggregationBuilder; - - @BeforeEach - void set_up() { - aggregationBuilder = new BucketAggregationBuilder(serializer); - } - - @Test - void should_build_bucket_with_field() { - assertEquals( - "{\n" - + " \"age\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"age\",\n" - + " \"size\" : 1000,\n" - + " \"min_doc_count\" : 1,\n" - + " \"shard_min_doc_count\" : 0,\n" - + " \"show_term_doc_count_error\" : false,\n" - + " \"order\" : {\n" - + " \"_key\" : \"asc\"\n" - + " }\n" - + " }\n" - + " }\n" - + "}", - buildQuery(named("age", ref("age", INTEGER)))); - } - - @Test - void should_build_bucket_with_literal() { - var literal = literal(1); - when(serializer.serialize(literal)).thenReturn("mock-serialize"); - assertEquals( - "{\n" - + " \"1\" : {\n" - + " \"terms\" : {\n" - + " \"script\" : {\n" - + " \"source\" :" - + " \"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"mock-serialize\\\"}\",\n" - + " \"lang\" : \"opensearch_compounded_script\"\n" - + " },\n" - + " \"size\" : 1000,\n" - + " \"min_doc_count\" : 1,\n" - + " \"shard_min_doc_count\" : 0,\n" - + " \"show_term_doc_count_error\" : false,\n" - + " \"order\" : {\n" - + " \"_key\" : \"asc\"\n" - + " }\n" - + " }\n" - + " }\n" - + "}", - buildQuery(named(literal))); - } - - @Test - void should_build_bucket_with_keyword_field() { - assertEquals( - "{\n" - + " \"name\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"name.keyword\",\n" - + " \"size\" : 1000,\n" - + " \"min_doc_count\" : 1,\n" - + " \"shard_min_doc_count\" : 0,\n" - + " \"show_term_doc_count_error\" : false,\n" - + " \"order\" : {\n" - + " \"_key\" : \"asc\"\n" - + " }\n" - + " }\n" - + " }\n" - + "}", - buildQuery( - named( - "name", - ref( - "name", - OpenSearchTextType.of( - Map.of( - "words", - OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword))))))); - } - - @Test - void should_build_bucket_with_parse_expression() { - ParseExpression parseExpression = - DSL.regex(ref("name.keyword", STRING), DSL.literal("(?\\w+)"), DSL.literal("name")); - when(serializer.serialize(parseExpression)).thenReturn("mock-serialize"); - assertEquals( - "{\n" - + " \"name\" : {\n" - + " \"terms\" : {\n" - + " \"script\" : {\n" - + " \"source\" :" - + " \"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"mock-serialize\\\"}\",\n" - + " \"lang\" : \"opensearch_compounded_script\"\n" - + " },\n" - + " \"size\" : 1000,\n" - + " \"min_doc_count\" : 1,\n" - + " \"shard_min_doc_count\" : 0,\n" - + " \"show_term_doc_count_error\" : false,\n" - + " \"order\" : {\n" - + " \"_key\" : \"asc\"\n" - + " }\n" - + " }\n" - + " }\n" - + "}", - buildQuery(named("name", parseExpression))); - } - - @Test - void terms_bucket_for_opensearchdate_type_uses_long() { - OpenSearchDateType dataType = OpenSearchDateType.of(ExprCoreType.TIMESTAMP); - - assertEquals( - "{\n" - + " \"date\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"date\",\n" - + " \"value_type\" : \"long\",\n" - + " \"size\" : 1000,\n" - + " \"min_doc_count\" : 1,\n" - + " \"shard_min_doc_count\" : 0,\n" - + " \"show_term_doc_count_error\" : false,\n" - + " \"order\" : {\n" - + " \"_key\" : \"asc\"\n" - + " }\n" - + " }\n" - + " }\n" - + "}", - buildQuery(named("date", ref("date", dataType)))); - } - - @Test - void terms_bucket_for_opensearchdate_type_uses_long_false() { - OpenSearchDateType dataType = OpenSearchDateType.of(STRING); - - assertEquals( - "{\n" - + " \"date\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"date\",\n" - + " \"size\" : 1000,\n" - + " \"min_doc_count\" : 1,\n" - + " \"shard_min_doc_count\" : 0,\n" - + " \"show_term_doc_count_error\" : false,\n" - + " \"order\" : {\n" - + " \"_key\" : \"asc\"\n" - + " }\n" - + " }\n" - + " }\n" - + "}", - buildQuery(named("date", ref("date", dataType)))); - } - - @ParameterizedTest(name = "{0}") - @EnumSource( - value = ExprCoreType.class, - names = {"TIMESTAMP", "TIME", "DATE", "DATETIME"}) - void terms_bucket_for_datetime_types_uses_long(ExprType dataType) { - assertEquals( - "{\n" - + " \"date\" : {\n" - + " \"terms\" : {\n" - + " \"field\" : \"date\",\n" - + " \"value_type\" : \"long\",\n" - + " \"size\" : 1000,\n" - + " \"min_doc_count\" : 1,\n" - + " \"shard_min_doc_count\" : 0,\n" - + " \"show_term_doc_count_error\" : false,\n" - + " \"order\" : {\n" - + " \"_key\" : \"asc\"\n" - + " }\n" - + " }\n" - + " }\n" - + "}", - buildQuery(named("date", ref("date", dataType)))); - } - - @SneakyThrows - private String buildQuery(NamedExpression groupByExpression) { - XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); - builder.startObject(); - ValuesSourceAggregationBuilder sourceBuilder = aggregationBuilder.build(groupByExpression); - sourceBuilder.toXContent(builder, EMPTY_PARAMS); - builder.endObject(); - return BytesReference.bytes(builder).utf8ToString(); - } -}