diff --git a/DEVELOPER_GUIDE.rst b/DEVELOPER_GUIDE.rst index 41be7cdbb9c..fbfb1fc5786 100644 --- a/DEVELOPER_GUIDE.rst +++ b/DEVELOPER_GUIDE.rst @@ -313,6 +313,7 @@ For test cases, you can use the cases in the following checklist in case you mis - *Explain* - DSL for simple query + - Script for complex expressions, see details in `intro-scripts <./docs/dev/intro-scripts.md>`_. - Execution plan for complex query like JOIN - *Response format* diff --git a/benchmarks/src/jmh/java/org/opensearch/sql/expression/operator/predicate/ExpressionScriptSerdeBenchmark.java b/benchmarks/src/jmh/java/org/opensearch/sql/expression/operator/predicate/ExpressionScriptSerdeBenchmark.java index 10ec42bf8d4..b46ac18f2c1 100644 --- a/benchmarks/src/jmh/java/org/opensearch/sql/expression/operator/predicate/ExpressionScriptSerdeBenchmark.java +++ b/benchmarks/src/jmh/java/org/opensearch/sql/expression/operator/predicate/ExpressionScriptSerdeBenchmark.java @@ -33,6 +33,7 @@ import org.opensearch.sql.expression.function.PPLFuncImpTable; import org.opensearch.sql.opensearch.storage.serde.DefaultExpressionSerializer; import org.opensearch.sql.opensearch.storage.serde.RelJsonSerializer; +import org.opensearch.sql.opensearch.storage.serde.ScriptParameterHelper; @Warmup(iterations = 1) @Measurement(iterations = 10) @@ -74,7 +75,9 @@ public void testRexNodeJsonSerde() { SqlStdOperatorTable.NOT_EQUALS, rexUpper, rexBuilder.makeLiteral("ABOUT")); Map fieldTypes = Map.of("Referer", ExprCoreType.STRING); - String serializedStr = relJsonSerializer.serialize(rexNotEquals, rowType, fieldTypes); + String serializedStr = + relJsonSerializer.serialize( + rexNotEquals, new ScriptParameterHelper(rowType.getFieldList(), fieldTypes)); relJsonSerializer.deserialize(serializedStr); } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java index ac0350e2815..c6d5a5cf552 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java @@ -68,9 +68,6 @@ public interface PlanUtils { String ROW_NUMBER_COLUMN_FOR_STREAMSTATS = "__stream_seq__"; String ROW_NUMBER_COLUMN_FOR_CHART = "_row_number_chart_"; - String DIRECTION = "DIRECTION"; - String NULL_DIRECTION = "NULL_DIRECTION"; - static SpanUnit intervalUnitToSpanUnit(IntervalUnit unit) { SpanUnit result; switch (unit) { diff --git a/docs/dev/intro-scripts.md b/docs/dev/intro-scripts.md new file mode 100644 index 00000000000..e4c3f9ad9a8 --- /dev/null +++ b/docs/dev/intro-scripts.md @@ -0,0 +1,75 @@ +# Scripts + +## Table of contents +- [V2 Scripts](#v2-scripts) +- [V3 Scripts](#v3-scripts) + +Script queries are generated when we push down some operators(e.g. `FILTER`, `AGGREGATE`, `SORT`) with functions that are not supported by OpenSearch DSL. + +We support script push down on both v2 and v3 engine, while their registered script languages and script format are different. + +## V2 Scripts + +V2 script is a v2 expression tree encoded in base64 string. It's a pure byte string content without any parameters. + +Example: + +For any query, it will generate a script query like below, with `V2_SCRIPT_ENCODED_BYTE_STRING` varies while it's also unreadable. + +```json +{ + "script": { + "source": "{\"langType\":\"v2\", \"script\":\"\"}", + "lang": "opensearch_compounded_script" + } +} +``` + +## V3 Scripts + +V3 script is a calcite expression tree encoded in base64 string. But before encoding, it has been standardized to make it reusable and then get higher cache hit ratio. As so, some specific information(like fields names, literal values) are extracted from the expression tree and put in the params field. + +Example1: + +Expression tree of `case(age < 35, 'u35' else email)` will generate a script query like below. And the expression tree will be standardized to `case(?0 < ?1, ?2 else ?3)` before encoding, in which `?i` indicates the i-th parameter. + +```json +{ + "script": { + "source": "{\"langType\":\"calcite\", \"script\":\"\"}", + "lang": "opensearch_compounded_script", + "params": { + "utcTimestamp": 17630261838681530000, + "SOURCES": [0, 2, 2, 1], + "DIGESTS": ["age", 35, "u35", "email"] + } + } +} +``` +There are usually 3 parts in the params: + +1. `utcTimestamp`: The timestamp when the query is executed, it's used by some time-related functions. +2. `SOURCES`: The source of the parameters, implies which source to retrieve the value for the i-th parameter. See class CalciteScriptEngine::Source, 0 stands for `DOC_VALUE`, 1 stands for `SOURCE`, 2 stands for `LITERAL`. +3. `DIGESTS`: The digest of each parameter, it's used as the key to retrieve the value from the corresponding sources. It will be field name for `DOC_VALUE` and `SOURCE`, while the literal value itself for `LITERAL`. + + +Example2: + +Sort script will add another specific parameter `MISSING_MAX`, which decides using MIN or MAX value for the missing value(i.e. NULL). The value of this parameter is derived from the combination of sort direction and null direction, see detail in SortExprDigest::isMissingMax. + +Sort command `sort (age + balance)` will generate a script query like below. + +```json +{ + "script": { + "source": "{\"langType\":\"calcite\", \"script\":\"\"}", + "lang": "opensearch_compounded_script", + "params": { + "MISSING_MAX": false, + "utcTimestamp": 17630261838681530000, + "SOURCES": [0, 0], + "DIGESTS": ["age", "balance"] + } + } +} +``` diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 416a5db3f1c..c209dc10e08 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -688,9 +688,9 @@ public void testExplainRegexMatchInWhereWithScriptPushdown() throws IOException enabledOnlyWhenPushdownIsEnabled(); String query = String.format("source=%s | where regexp_match(name, 'hello')", TEST_INDEX_STRINGS); - var result = explainQueryToString(query); - String expected = loadFromFile("expectedOutput/calcite/explain_regexp_match_in_where.json"); - assertJsonEqualsIgnoreId(expected, result); + var result = explainQueryYaml(query); + String expected = loadFromFile("expectedOutput/calcite/explain_regexp_match_in_where.yaml"); + assertYamlEqualsIgnoreId(expected, result); } @Test @@ -1417,9 +1417,9 @@ public void testStrftimeFunctionExplain() throws IOException { // Script generation is not stable in v2 @Test public void testExplainPushDownScriptsContainingUDT() throws IOException { - assertJsonEqualsIgnoreId( - loadExpectedPlan("explain_filter_script_ip_push.json"), - explainQueryToString( + assertYamlEqualsIgnoreId( + loadExpectedPlan("explain_filter_script_ip_push.yaml"), + explainQueryYaml( String.format( "source=%s | where cidrmatch(host, '0.0.0.0/24') | fields host", TEST_INDEX_WEBLOGS))); diff --git a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java index fe5520e99ac..1e3e40e112e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java @@ -408,7 +408,8 @@ private static String eliminatePid(String s) { public static void assertYamlEqualsIgnoreId(String expectedYaml, String actualYaml) { String cleanedYaml = cleanUpYaml(actualYaml); - assertYamlEquals(expectedYaml, cleanedYaml); + String cleanedExpectedYaml = cleanUpYaml(expectedYaml); + assertYamlEquals(cleanedExpectedYaml, cleanedYaml); } public static void assertYamlEquals(String expected, String actual) { @@ -426,7 +427,8 @@ private static String cleanUpYaml(String s) { return s.replaceAll("\"utcTimestamp\":\\d+", "\"utcTimestamp\": 0") .replaceAll("rel#\\d+", "rel#") .replaceAll("RelSubset#\\d+", "RelSubset#") - .replaceAll("pitId=[^,]+,", "pitId=*,"); + .replaceAll("pitId=[^,]+,", "pitId=*,") + .replaceAll("\\$t?\\d+", "\\$FIELD_INDEX"); } private static String jsonToYaml(String json) { diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_case_cannot_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_case_cannot_push.yaml index d04bbd2df44..c9929d83e81 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/agg_case_cannot_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_case_cannot_push.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[30..40]]), 'u40':VARCHAR, 'u100':VARCHAR)], age=[$10]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg_age=AVG($1)), PROJECT->[avg_age, age_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age_range":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BXZ7CiAgIm9wIjogewogICAgIm5hbWUiOiAiQ0FTRSIsCiAgICAia2luZCI6ICJDQVNFIiwKICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIjwiLAogICAgICAgICJraW5kIjogIkxFU1NfVEhBTiIsCiAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgIm5hbWUiOiAiJDAiCiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAibGl0ZXJhbCI6IDMwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAidTMwIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJTRUFSQ0giLAogICAgICAgICJraW5kIjogIlNFQVJDSCIsCiAgICAgICAgInN5bnRheCI6ICJJTlRFUk5BTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogewogICAgICAgICAgICAicmFuZ2VTZXQiOiBbCiAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgImNsb3NlZCIsCiAgICAgICAgICAgICAgICAiMzAiLAogICAgICAgICAgICAgICAgIjQwIgogICAgICAgICAgICAgIF0KICAgICAgICAgICAgXSwKICAgICAgICAgICAgIm51bGxBcyI6ICJVTktOT1dOIgogICAgICAgICAgfSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogInU0MCIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICJ1MTAwIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg_age=AVG($1)), PROJECT->[avg_age, age_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age_range":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQGFHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiPCIsCiAgICAgICAgImtpbmQiOiAiTEVTU19USEFOIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlNFQVJDSCIsCiAgICAgICAgImtpbmQiOiAiU0VBUkNIIiwKICAgICAgICAic3ludGF4IjogIklOVEVSTkFMIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDMsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogewogICAgICAgICAgICAicmFuZ2VTZXQiOiBbCiAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgImNsb3NlZCIsCiAgICAgICAgICAgICAgICAiMzAiLAogICAgICAgICAgICAgICAgIjQwIgogICAgICAgICAgICAgIF0KICAgICAgICAgICAgXSwKICAgICAgICAgICAgIm51bGxBcyI6ICJVTktOT1dOIgogICAgICAgICAgfSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiA0LAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDUsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2,0,2,2],"DIGESTS":["age",30,"u30","age","u40","u100"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_case_composite_cannot_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_case_composite_cannot_push.yaml index 82cbadeb735..20199016fb4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/agg_case_composite_cannot_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_case_composite_cannot_push.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(age_range=[CASE(<($10, 35), 'u35':VARCHAR, $11)], state=[$9], balance=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2)), PROJECT->[avg_balance, age_range, state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age_range":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQA5nsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQCe3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiPCIsCiAgICAgICAgImtpbmQiOiAiTEVTU19USEFOIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogMzUsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICJ1MzUiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImlucHV0IjogMSwKICAgICAgIm5hbWUiOiAiJDEiCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAACdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHSU5URUdFUnQABWVtYWlsc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+ABB4cH5xAH4ACnQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgALdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AG3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AHQAAAABzcQB+AAAAAAADdwQAAAAAeHh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2)), PROJECT->[avg_balance, age_range, state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age_range":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQDJXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiPCIsCiAgICAgICAgImtpbmQiOiAiTEVTU19USEFOIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDMsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2,1],"DIGESTS":["age",35,"u35","email"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_case_num_res_cannot_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_case_num_res_cannot_push.yaml index 9502c66a448..f50f5e44582 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/agg_case_num_res_cannot_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_case_num_res_cannot_push.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(age_range=[CASE(<($10, 30), 30, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), age_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age_range":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAe3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0Ap17CiAgIm9wIjogewogICAgIm5hbWUiOiAiQ0FTRSIsCiAgICAia2luZCI6ICJDQVNFIiwKICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIjwiLAogICAgICAgICJraW5kIjogIkxFU1NfVEhBTiIsCiAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgIm5hbWUiOiAiJDAiCiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAibGl0ZXJhbCI6IDMwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAzMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAxMDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANhZ2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdJTlRFR0VSeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"value_type":"long","missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), age_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age_range":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQC9HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiPCIsCiAgICAgICAgImtpbmQiOiAiTEVTU19USEFOIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMywKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2,2],"DIGESTS":["age",30,30,100]}},"missing_bucket":true,"value_type":"long","missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q19.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q19.yaml index 88a57952c89..e0c99960471 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q19.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q19.yaml @@ -9,4 +9,4 @@ calcite: LogicalProject(EventDate=[$0], URLRegionID=[$1], HasGCLID=[$2], Income=[$3], Interests=[$4], Robotness=[$5], BrowserLanguage=[$6], CounterClass=[$7], BrowserCountry=[$8], OriginalURL=[$9], ClientTimeZone=[$10], RefererHash=[$11], TraficSourceID=[$12], HitColor=[$13], RefererRegionID=[$14], URLCategoryID=[$15], LocalEventTime=[$16], EventTime=[$17], UTMTerm=[$18], AdvEngineID=[$19], UserAgentMinor=[$20], UserAgentMajor=[$21], RemoteIP=[$22], Sex=[$23], JavaEnable=[$24], URLHash=[$25], URL=[$26], ParamOrderID=[$27], OpenstatSourceID=[$28], HTTPError=[$29], SilverlightVersion3=[$30], MobilePhoneModel=[$31], SilverlightVersion4=[$32], SilverlightVersion1=[$33], SilverlightVersion2=[$34], IsDownload=[$35], IsParameter=[$36], CLID=[$37], FlashMajor=[$38], FlashMinor=[$39], UTMMedium=[$40], WatchID=[$41], DontCountHits=[$42], CookieEnable=[$43], HID=[$44], SocialAction=[$45], WindowName=[$46], ConnectTiming=[$47], PageCharset=[$48], IsLink=[$49], IsArtifical=[$50], JavascriptEnable=[$51], ClientEventTime=[$52], DNSTiming=[$53], CodeVersion=[$54], ResponseEndTiming=[$55], FUniqID=[$56], WindowClientHeight=[$57], OpenstatServiceName=[$58], UTMContent=[$59], HistoryLength=[$60], IsOldCounter=[$61], MobilePhone=[$62], SearchPhrase=[$63], FlashMinor2=[$64], SearchEngineID=[$65], IsEvent=[$66], UTMSource=[$67], RegionID=[$68], OpenstatAdID=[$69], UTMCampaign=[$70], GoodEvent=[$71], IsRefresh=[$72], ParamCurrency=[$73], Params=[$74], ResolutionHeight=[$75], ClientIP=[$76], FromTag=[$77], ParamCurrencyID=[$78], ResponseStartTiming=[$79], ResolutionWidth=[$80], SendTiming=[$81], RefererCategoryID=[$82], OpenstatCampaignID=[$83], UserID=[$84], WithHash=[$85], UserAgent=[$86], ParamPrice=[$87], ResolutionDepth=[$88], IsMobile=[$89], Age=[$90], SocialSourceNetworkID=[$91], OpenerName=[$92], OS=[$93], IsNotBounce=[$94], Referer=[$95], NetMinor=[$96], Title=[$97], NetMajor=[$98], IPNetworkID=[$99], FetchTiming=[$100], SocialNetwork=[$101], SocialSourcePage=[$102], CounterID=[$103], WindowClientWidth=[$104], _id=[$105], _index=[$106], _score=[$107], _maxscore=[$108], _sort=[$109], _routing=[$110], m=[EXTRACT('minute':VARCHAR, $17)]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},count()=COUNT()), PROJECT->[UserID, m, SearchPhrase, count()], SORT_AGG_METRICS->[3 DESC LAST], PROJECT->[count(), UserID, m, SearchPhrase], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"SearchPhrase|UserID|m":{"multi_terms":{"terms":[{"field":"SearchPhrase"},{"field":"UserID"},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAt3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiRXZlbnRUaW1lIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0Ae57CiAgIm9wIjogewogICAgIm5hbWUiOiAiRVhUUkFDVCIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAibGl0ZXJhbCI6ICJtaW51dGUiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQACUV2ZW50VGltZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGVUeXBlni1SrhB9yq8CAAFMAAdmb3JtYXRzdAAQTGphdmEvdXRpbC9MaXN0O3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzdAAPTGphdmEvdXRpbC9NYXA7eHB+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAlUSU1FU1RBTVB+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgASdAAERGF0ZXNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGXhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGwAAAABzcgATamF2YS51dGlsLkFycmF5TGlzdHiB0h2Zx2GdAwABSQAEc2l6ZXhwAAAAA3cEAAAAA3QAE3l5eXktTU0tZGQgSEg6bW06c3N0ABlzdHJpY3RfZGF0ZV9vcHRpb25hbF90aW1ldAAMZXBvY2hfbWlsbGlzeHh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"value_type":"long"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"count()":"desc"},{"_key":"asc"}]},"aggregations":{"count()":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},count()=COUNT()), PROJECT->[UserID, m, SearchPhrase, count()], SORT_AGG_METRICS->[3 DESC LAST], PROJECT->[count(), UserID, m, SearchPhrase], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"SearchPhrase|UserID|m":{"multi_terms":{"terms":[{"field":"SearchPhrase"},{"field":"UserID"},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCZnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJFWFRSQUNUIiwKICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ1ZHQiOiAiRVhQUl9USU1FU1RBTVAiLAogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdLAogICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAibnVsbGFibGUiOiB0cnVlCiAgfSwKICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgImR5bmFtaWMiOiBmYWxzZQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[2,0],"DIGESTS":["minute","EventTime"]}},"value_type":"long"}],"size":10,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"count()":"desc"},{"_key":"asc"}]},"aggregations":{"count()":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q28.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q28.yaml index 838a201cf92..daa53e1c368 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q28.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q28.yaml @@ -14,4 +14,4 @@ calcite: EnumerableLimit(fetch=[25]) EnumerableSort(sort0=[$0], dir0=[DESC-nulls-last]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[100000], expr#4=[>($t1, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[URL, CounterID], FILTER->AND(<>($0, ''), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},l=AVG($1),c=COUNT()), PROJECT->[l, c, CounterID]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"URL","boost":1.0}}],"must_not":[{"term":{"URL":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"CounterID","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["URL","CounterID"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"CounterID":{"terms":{"field":"CounterID","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"l":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAknsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJVUkwiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQApnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiAwLAogICAgICAibmFtZSI6ICIkMCIKICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AANVUkx+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAZTVFJJTkd4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[URL, CounterID], FILTER->AND(<>($0, ''), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},l=AVG($1),c=COUNT()), PROJECT->[l, c, CounterID]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"URL","boost":1.0}}],"must_not":[{"term":{"URL":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"CounterID","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["URL","CounterID"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"CounterID":{"terms":{"field":"CounterID","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"l":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["URL"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q29.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q29.yaml index 9a7638f1811..4ec2ad5e6c8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q29.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q29.yaml @@ -15,4 +15,4 @@ calcite: EnumerableLimit(fetch=[25]) EnumerableSort(sort0=[$0], dir0=[DESC-nulls-last]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100000], expr#5=[>($t1, $t4)], proj#0..3=[{exprs}], $condition=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[Referer], SCRIPT->AND(<>($0, ''), IS NOT NULL(REGEXP_REPLACE($0, '^https?://(?:www\.)?([^/]+)/.*$':VARCHAR, '$1'))), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},l=AVG($2),c=COUNT(),min(Referer)=MIN($1)), PROJECT->[l, c, min(Referer), k]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"Referer","boost":1.0}}],"must_not":[{"term":{"Referer":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJSZWZlcmVyIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AsN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiUkVHRVhQX1JFUExBQ0UiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiXmh0dHBzPzovLyg/Ond3d1xcLik/KFteL10rKS8uKiQiLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAibGl0ZXJhbCI6ICIkMSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMgogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQAB1JlZmVyZXJ+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAZTVFJJTkd4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["Referer"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"k":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJSZWZlcmVyIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0Adp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiUkVHRVhQX1JFUExBQ0UiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICJeaHR0cHM/Oi8vKD86d3d3XFwuKT8oW14vXSspLy4qJCIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICIkMSIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAyCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAdSZWZlcmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAGU1RSSU5HeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":false,"order":"asc"}}}]},"aggregations":{"l":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJSZWZlcmVyIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AKZ7CiAgIm9wIjogewogICAgIm5hbWUiOiAiQ0hBUl9MRU5HVEgiLAogICAgImtpbmQiOiAiQ0hBUl9MRU5HVEgiLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAAHUmVmZXJlcn5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABlNUUklOR3h4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}},"min(Referer)":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"Referer"}],"sort":[{"Referer":{"order":"asc"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[Referer], SCRIPT->AND(<>($0, ''), IS NOT NULL(REGEXP_REPLACE($0, '^https?://(?:www\.)?([^/]+)/.*$':VARCHAR, '$1'))), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},l=AVG($2),c=COUNT(),min(Referer)=MIN($1)), PROJECT->[l, c, min(Referer), k]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"Referer","boost":1.0}}],"must_not":[{"term":{"Referer":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQDEnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJJUyBOT1QgTlVMTCIsCiAgICAia2luZCI6ICJJU19OT1RfTlVMTCIsCiAgICAic3ludGF4IjogIlBPU1RGSVgiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMgogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["Referer","^https?://(?:www\\.)?([^/]+)/.*$","$1"]}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["Referer"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"k":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCGXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IDIKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["Referer","^https?://(?:www\\.)?([^/]+)/.*$","$1"]}},"missing_bucket":false,"order":"asc"}}}]},"aggregations":{"l":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["Referer"]}}}},"min(Referer)":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"Referer"}],"sort":[{"Referer":{"order":"asc"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q40.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q40.yaml index 3c74a22c40f..00d724db076 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q40.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q40.yaml @@ -13,4 +13,4 @@ calcite: EnumerableCalc(expr#0..5=[{inputs}], PageViews=[$t5], TraficSourceID=[$t0], SearchEngineID=[$t2], AdvEngineID=[$t1], Src=[$t3], Dst=[$t4]) EnumerableLimit(fetch=[10000]) EnumerableLimit(offset=[1000], fetch=[10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventDate, TraficSourceID, AdvEngineID, URL, SearchEngineID, IsRefresh, Referer, CounterID], FILTER->AND(=($7, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':VARCHAR..'2013-07-31 00:00:00':VARCHAR]]:VARCHAR), =($5, 0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2, 3, 4},PageViews=COUNT()), SORT_AGG_METRICS->[5 DESC LAST], LIMIT->[10 from 1000]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-31T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"term":{"IsRefresh":{"value":0,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["EventDate","TraficSourceID","AdvEngineID","URL","SearchEngineID","IsRefresh","Referer","CounterID"],"excludes":[]},"aggregations":{"TraficSourceID|AdvEngineID|SearchEngineID|Src|Dst":{"multi_terms":{"terms":[{"field":"TraficSourceID"},{"field":"AdvEngineID"},{"field":"SearchEngineID"},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQBT3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJTTUFMTElOVCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIlNlYXJjaEVuZ2luZUlEIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiU01BTExJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJBZHZFbmdpbmVJRCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIlJlZmVyZXIiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQE8XsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiQU5EIiwKICAgICAgICAia2luZCI6ICJBTkQiLAogICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgIm9wIjogewogICAgICAgICAgICAibmFtZSI6ICI9IiwKICAgICAgICAgICAgImtpbmQiOiAiRVFVQUxTIiwKICAgICAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgICAgICB9LAogICAgICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICAgICAgfSwKICAgICAgICAgICAgewogICAgICAgICAgICAgICJsaXRlcmFsIjogMCwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiPSIsCiAgICAgICAgICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAgICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICAgICAgfSwKICAgICAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICAgICAgewogICAgICAgICAgICAgICJpbnB1dCI6IDEsCiAgICAgICAgICAgICAgIm5hbWUiOiAiJDEiCiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAibGl0ZXJhbCI6IDAsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9CiAgICAgICAgICBdCiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAiaW5wdXQiOiAyLAogICAgICAibmFtZSI6ICIkMiIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIiIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAADdAAOU2VhcmNoRW5naW5lSUR+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAVTSE9SVHQAB1JlZmVyZXJ+cQB+AAp0AAZTVFJJTkd0AAtBZHZFbmdpbmVJRHEAfgAMeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}},{"field":"URL"}],"size":1010,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"PageViews":"desc"},{"_key":"asc"}]},"aggregations":{"PageViews":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventDate, TraficSourceID, AdvEngineID, URL, SearchEngineID, IsRefresh, Referer, CounterID], FILTER->AND(=($7, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':VARCHAR..'2013-07-31 00:00:00':VARCHAR]]:VARCHAR), =($5, 0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2, 3, 4},PageViews=COUNT()), SORT_AGG_METRICS->[5 DESC LAST], LIMIT->[10 from 1000]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-31T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"term":{"IsRefresh":{"value":0,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["EventDate","TraficSourceID","AdvEngineID","URL","SearchEngineID","IsRefresh","Referer","CounterID"],"excludes":[]},"aggregations":{"TraficSourceID|AdvEngineID|SearchEngineID|Src|Dst":{"multi_terms":{"terms":[{"field":"TraficSourceID"},{"field":"AdvEngineID"},{"field":"SearchEngineID"},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQGCnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiQU5EIiwKICAgICAgICAia2luZCI6ICJBTkQiLAogICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgIm9wIjogewogICAgICAgICAgICAibmFtZSI6ICI9IiwKICAgICAgICAgICAgImtpbmQiOiAiRVFVQUxTIiwKICAgICAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgICAgICB9LAogICAgICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJTTUFMTElOVCIsCiAgICAgICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9CiAgICAgICAgICBdCiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAib3AiOiB7CiAgICAgICAgICAgICJuYW1lIjogIj0iLAogICAgICAgICAgICAia2luZCI6ICJFUVVBTFMiLAogICAgICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgICAgIH0sCiAgICAgICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIlNNQUxMSU5UIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMywKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0KICAgICAgICAgIF0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiA0LAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogNSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,0,2,0,2],"DIGESTS":["SearchEngineID",0,"AdvEngineID",0,"Referer",""]}}},{"field":"URL"}],"size":1010,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"PageViews":"desc"},{"_key":"asc"}]},"aggregations":{"PageViews":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q43.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q43.yaml index 019cd05cdac..b2a68c85d2f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q43.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q43.yaml @@ -13,4 +13,4 @@ calcite: EnumerableCalc(expr#0..1=[{inputs}], PageViews=[$t1], M=[$t0]) EnumerableLimit(fetch=[10000]) EnumerableLimit(offset=[1000], fetch=[10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventDate, EventTime, DontCountHits, IsRefresh, CounterID], FILTER->AND(=($4, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':VARCHAR..'2013-07-15 00:00:00':VARCHAR]]:VARCHAR), =($3, 0), =($2, 0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},PageViews=COUNT()), SORT->[0 ASC FIRST], LIMIT->[10 from 1000]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-15T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"term":{"IsRefresh":{"value":0,"boost":1.0}}},{"term":{"DontCountHits":{"value":0,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["EventDate","EventTime","DontCountHits","IsRefresh","CounterID"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1010,"sources":[{"M":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAt3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiRXZlbnRUaW1lIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AhN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiREFURV9GT1JNQVQiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICIlWS0lbS0lZCAlSDowMDowMCIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAicHJlY2lzaW9uIjogLTEKICB9LAogICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAiZHluYW1pYyI6IGZhbHNlCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAlFdmVudFRpbWVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRlVHlwZZ4tUq4QfcqvAgABTAAHZm9ybWF0c3QAEExqYXZhL3V0aWwvTGlzdDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3QAD0xqYXZhL3V0aWwvTWFwO3hwfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAJVElNRVNUQU1QfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4AEnQABERhdGVzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABl4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+ABsAAAAAc3IAE2phdmEudXRpbC5BcnJheUxpc3R4gdIdmcdhnQMAAUkABHNpemV4cAAAAAN3BAAAAAN0ABN5eXl5LU1NLWRkIEhIOm1tOnNzdAAZc3RyaWN0X2RhdGVfb3B0aW9uYWxfdGltZXQADGVwb2NoX21pbGxpc3h4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventDate, EventTime, DontCountHits, IsRefresh, CounterID], FILTER->AND(=($4, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':VARCHAR..'2013-07-15 00:00:00':VARCHAR]]:VARCHAR), =($3, 0), =($2, 0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},PageViews=COUNT()), SORT->[0 ASC FIRST], LIMIT->[10 from 1000]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-15T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"term":{"IsRefresh":{"value":0,"boost":1.0}}},{"term":{"DontCountHits":{"value":0,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["EventDate","EventTime","DontCountHits","IsRefresh","CounterID"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1010,"sources":[{"M":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCgHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJEQVRFX0ZPUk1BVCIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInVkdCI6ICJFWFBSX1RJTUVTVEFNUCIsCiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAicHJlY2lzaW9uIjogLTEKICB9LAogICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAiZHluYW1pYyI6IGZhbHNlCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["EventTime","%Y-%m-%d %H:00:00"]}},"missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml index 9d452158542..b0b23b7ce10 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(gender=[$4], b_1=[+($3, 1)], $f3=[POWER($3, 2)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(b_1)=COUNT($1),c3=COUNT($2)), PROJECT->[count(b_1), c3, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(b_1)":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAfnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AQF7CiAgIm9wIjogewogICAgIm5hbWUiOiAiKyIsCiAgICAia2luZCI6ICJQTFVTIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiAwLAogICAgICAibmFtZSI6ICIkMCIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQAB2JhbGFuY2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05HeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}},"c3":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAfnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0ARF7CiAgIm9wIjogewogICAgIm5hbWUiOiAiUE9XRVIiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAdiYWxhbmNlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3h4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(b_1)=COUNT($1),c3=COUNT($2)), PROJECT->[count(b_1), c3, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(b_1)":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBRXsKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",1]}}}},"c3":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBVXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJQT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",2]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_timestamp_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_timestamp_push.yaml index bfaf02ad47c..03ae462f020 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_timestamp_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_timestamp_push.yaml @@ -7,4 +7,4 @@ calcite: LogicalProject(t=[UNIX_TIMESTAMP($3)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), t], SORT->[1 ASC FIRST], LIMIT->3, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":3,"sources":[{"t":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAt3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AWd7CiAgIm9wIjogewogICAgIm5hbWUiOiAiVU5JWF9USU1FU1RBTVAiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJET1VCTEUiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQACWJpcnRoZGF0ZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGVUeXBlni1SrhB9yq8CAAFMAAdmb3JtYXRzdAAQTGphdmEvdXRpbC9MaXN0O3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzdAAPTGphdmEvdXRpbC9NYXA7eHB+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAlUSU1FU1RBTVB+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgASdAAERGF0ZXNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGXhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGwAAAABzcQB+AAAAAAABdwQAAAAAeHh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"value_type":"double","missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), t], SORT->[1 ASC FIRST], LIMIT->3, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":3,"sources":[{"t":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQB4XsKICAib3AiOiB7CiAgICAibmFtZSI6ICJVTklYX1RJTUVTVEFNUCIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInVkdCI6ICJFWFBSX1RJTUVTVEFNUCIsCiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0sCiAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9LAogICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAiZHluYW1pYyI6IGZhbHNlCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["birthdate"]}},"missing_bucket":true,"value_type":"double","missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.yaml index 0c34a565cdc..ae68535cb0b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.yaml @@ -8,4 +8,4 @@ calcite: LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], t=[DATE_ADD($3, 1:INTERVAL DAY)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[SCRIPT->IS NOT NULL(DATE_ADD($3, 1:INTERVAL DAY)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(t,1d)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAt3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AyV7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiREFURV9BREQiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVSVkFMX0RBWSIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMTAsCiAgICAgICAgICAgICJzY2FsZSI6IDYKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInVkdCI6ICJFWFBSX1RJTUVTVEFNUCIsCiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfSwKICAgICAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAAJYmlydGhkYXRlc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0ZVR5cGWeLVKuEH3KrwIAAUwAB2Zvcm1hdHN0ABBMamF2YS91dGlsL0xpc3Q7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXN0AA9MamF2YS91dGlsL01hcDt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQACVRJTUVTVEFNUH5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+ABJ0AAREYXRlc3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAZeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAbAAAAAHNxAH4AAAAAAAF3BAAAAAB4eHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(t,1d)":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAt3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0BQN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiU1BBTiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiREFURV9BREQiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVSVkFMX0RBWSIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMTAsCiAgICAgICAgICAgICJzY2FsZSI6IDYKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInVkdCI6ICJFWFBSX1RJTUVTVEFNUCIsCiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfSwKICAgICAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogImQiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogMQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICJwcmVjaXNpb24iOiAtMQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQACWJpcnRoZGF0ZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGVUeXBlni1SrhB9yq8CAAFMAAdmb3JtYXRzdAAQTGphdmEvdXRpbC9MaXN0O3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzdAAPTGphdmEvdXRpbC9NYXA7eHB+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAlUSU1FU1RBTVB+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgASdAAERGF0ZXNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGXhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGwAAAABzcQB+AAAAAAABdwQAAAAAeHh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":false,"value_type":"long","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[SCRIPT->IS NOT NULL(DATE_ADD($3, 1:INTERVAL DAY)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(t,1d)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQDs3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJJUyBOT1QgTlVMTCIsCiAgICAia2luZCI6ICJJU19OT1RfTlVMTCIsCiAgICAic3ludGF4IjogIlBPU1RGSVgiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJEQVRFX0FERCIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVSVkFMX0RBWSIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMTAsCiAgICAgICAgICAgICJzY2FsZSI6IDYKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInVkdCI6ICJFWFBSX1RJTUVTVEFNUCIsCiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfSwKICAgICAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["birthdate"]}},"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(t,1d)":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQFmXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJTUEFOIiwKICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJEQVRFX0FERCIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVSVkFMX0RBWSIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMTAsCiAgICAgICAgICAgICJzY2FsZSI6IDYKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInVkdCI6ICJFWFBSX1RJTUVTVEFNUCIsCiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfSwKICAgICAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAxCiAgICAgIH0KICAgIH0KICBdLAogICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAidHlwZSI6IHsKICAgICJ1ZHQiOiAiRVhQUl9USU1FU1RBTVAiLAogICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAibnVsbGFibGUiOiB0cnVlLAogICAgInByZWNpc2lvbiI6IC0xCiAgfSwKICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgImR5bmFtaWMiOiBmYWxzZQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["birthdate",1,"d"]}},"missing_bucket":false,"value_type":"long","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml index 1a3df86b66a..0bc8e26bdd2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_complex2.yaml @@ -9,4 +9,4 @@ calcite: LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_state=[LOWER($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1, 3},sum(balance)=SUM($0),count()=COUNT(),d=COUNT(DISTINCT $2)), SORT_AGG_METRICS->[4 DESC LAST], PROJECT->[sum(balance), count(), d, gender, new_state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"gender|new_state":{"multi_terms":{"terms":[{"field":"gender.keyword"},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlHsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJzdGF0ZSIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IGZhbHNlCn10AARleHBydACjewogICJvcCI6IHsKICAgICJuYW1lIjogIkxPV0VSIiwKICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDAsCiAgICAgICJuYW1lIjogIiQwIgogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQABXN0YXRlc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4ADH5xAH4AEHQABlNUUklOR35xAH4AFHQAB0tleXdvcmRxAH4AGXh4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}],"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"d":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"d":{"cardinality":{"field":"employer.keyword"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1, 3},sum(balance)=SUM($0),count()=COUNT(),d=COUNT(DISTINCT $2)), SORT_AGG_METRICS->[4 DESC LAST], PROJECT->[sum(balance), count(), d, gender, new_state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"gender|new_state":{"multi_terms":{"terms":[{"field":"gender.keyword"},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}}}],"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"d":"desc"},{"_key":"asc"}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"d":{"cardinality":{"field":"employer.keyword"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_terms_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_terms_script.yaml index b020c1003b1..ddbe8b2764e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_terms_script.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_multi_terms_script.yaml @@ -9,4 +9,4 @@ calcite: LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_gender=[LOWER($4)], new_state=[LOWER($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), SORT_AGG_METRICS->[2 ASC FIRST], PROJECT->[count(), new_gender, new_state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"new_gender|new_state":{"multi_terms":{"terms":[{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJnZW5kZXIiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQAo3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiAwLAogICAgICAibmFtZSI6ICIkMCIKICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAZnZW5kZXJzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AC3hwfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+ABF0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAYeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAaAAAAAHNxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgAMfnEAfgAQdAAGU1RSSU5HfnEAfgAUdAAHS2V5d29yZHEAfgAZeHh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlHsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJzdGF0ZSIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IGZhbHNlCn10AARleHBydACjewogICJvcCI6IHsKICAgICJuYW1lIjogIkxPV0VSIiwKICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDAsCiAgICAgICJuYW1lIjogIiQwIgogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQABXN0YXRlc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4ADH5xAH4AEHQABlNUUklOR35xAH4AFHQAB0tleXdvcmRxAH4AGXh4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}],"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"count()":"asc"},{"_key":"asc"}]},"aggregations":{"count()":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), SORT_AGG_METRICS->[2 ASC FIRST], PROJECT->[count(), new_gender, new_state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"new_gender|new_state":{"multi_terms":{"terms":[{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["gender.keyword"]}}},{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}}}],"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"count()":"asc"},{"_key":"asc"}]},"aggregations":{"count()":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_script.yaml index 52852044d92..9292b81e5be 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_script.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_measure_script.yaml @@ -9,4 +9,4 @@ calcite: LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], new_state=[LOWER($7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT_AGG_METRICS->[1 ASC FIRST], PROJECT->[count(), new_state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlHsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJzdGF0ZSIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IGZhbHNlCn10AARleHBydACjewogICJvcCI6IHsKICAgICJuYW1lIjogIkxPV0VSIiwKICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDAsCiAgICAgICJuYW1lIjogIiQwIgogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQABXN0YXRlc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4ADH5xAH4AEHQABlNUUklOR35xAH4AFHQAB0tleXdvcmRxAH4AGXh4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"count()":"asc"},{"_key":"asc"}]},"aggregations":{"count()":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT_AGG_METRICS->[1 ASC FIRST], PROJECT->[count(), new_state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"new_state":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJMT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["state.keyword"]}},"size":1000,"min_doc_count":1,"shard_min_doc_count":0,"show_term_doc_count_error":false,"order":[{"count()":"asc"},{"_key":"asc"}]},"aggregations":{"count()":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml index fcbc002565c..0726597ef90 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CHAR_LENGTH($t0)], sum=[$t1], len=[$t2], gender=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum=SUM($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAfnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AQN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiKyIsCiAgICAia2luZCI6ICJQTFVTIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiAwLAogICAgICAibmFtZSI6ICIkMCIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogMTAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAAHYmFsYW5jZX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum=SUM($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBRXsKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml index e87eb6d895c..8fd23589e33 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAfnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0Ac97CiAgIm9wIjogewogICAgIm5hbWUiOiAiRElWSURFIiwKICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDAsCiAgICAgICJuYW1lIjogIiQwIgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAxMDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdLAogICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAibnVsbGFibGUiOiB0cnVlCiAgfSwKICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgImR5bmFtaWMiOiBmYWxzZQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAAHYmFsYW5jZX5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCEXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0sCiAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiQklHSU5UIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9LAogICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAiZHluYW1pYyI6IGZhbHNlCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml index 6461c40061c..482e7d299e7 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_no_expr_output_push.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, $7)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[SORT_EXPR->[+($10, $7) ASCENDING NULLS_FIRST], LIMIT->10000, PROJECT->[age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQA0HsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IGZhbHNlCn10AARleHBydADFewogICJvcCI6IHsKICAgICJuYW1lIjogIisiLAogICAgImtpbmQiOiAiUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9LAogICAgewogICAgICAiaW5wdXQiOiAxLAogICAgICAibmFtZSI6ICIkMSIKICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAJ0AAdiYWxhbmNlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QAA2FnZX5xAH4ACnQAB0lOVEVHRVJ4eA==\"}","lang":"opensearch_compounded_script","params":{"NULL_DIRECTION":"FIRST","DIRECTION":"ASCENDING","utcTimestamp": 0}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[SORT_EXPR->[+($10, $7) ASCENDING NULLS_FIRST], LIMIT->10000, PROJECT->[age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBRHsKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml index 256c393b069..77ebbda2729 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_project_then_sort.yaml @@ -6,4 +6,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], expr#2=[+($t0, $t1)], age=[$t0], $f1=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+($0, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQA0HsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IGZhbHNlCn10AARleHBydADFewogICJvcCI6IHsKICAgICJuYW1lIjogIisiLAogICAgImtpbmQiOiAiUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9LAogICAgewogICAgICAiaW5wdXQiOiAxLAogICAgICAibmFtZSI6ICIkMSIKICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAJ0AAdiYWxhbmNlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QAA2FnZX5xAH4ACnQAB0lOVEVHRVJ4eA==\"}","lang":"opensearch_compounded_script","params":{"NULL_DIRECTION":"FIRST","DIRECTION":"ASCENDING","utcTimestamp": 0}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+($0, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBRHsKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml index b509fb3117f..8762b609c0b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_push.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], expr#2=[+($t0, $t1)], age=[$t0], $f1=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+($0, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQA0HsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IGZhbHNlCn10AARleHBydADFewogICJvcCI6IHsKICAgICJuYW1lIjogIisiLAogICAgImtpbmQiOiAiUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9LAogICAgewogICAgICAiaW5wdXQiOiAxLAogICAgICAibmFtZSI6ICIkMSIKICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAJ0AAdiYWxhbmNlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QAA2FnZX5xAH4ACnQAB0lOVEVHRVJ4eA==\"}","lang":"opensearch_compounded_script","params":{"NULL_DIRECTION":"FIRST","DIRECTION":"ASCENDING","utcTimestamp": 0}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+($0, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBRHsKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml index db832db7b4d..b71697ce1dd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_single_expr_output_push.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], expr#2=[+($t0, $t1)], $f0=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+($0, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQA0HsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IGZhbHNlCn10AARleHBydADFewogICJvcCI6IHsKICAgICJuYW1lIjogIisiLAogICAgImtpbmQiOiAiUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9LAogICAgewogICAgICAiaW5wdXQiOiAxLAogICAgICAibmFtZSI6ICIkMSIKICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAJ0AAdiYWxhbmNlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QAA2FnZX5xAH4ACnQAB0lOVEVHRVJ4eA==\"}","lang":"opensearch_compounded_script","params":{"NULL_DIRECTION":"FIRST","DIRECTION":"ASCENDING","utcTimestamp": 0}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age, balance], SORT_EXPR->[+($0, $1) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["age","balance"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBRHsKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml index b57328afbae..91d013e1536 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_nested_expr.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..12=[{inputs}], expr#13=[+($t10, $t7)], expr#14=[-($t13, $t10)], proj#0..14=[{exprs}]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[-(+($10, $7), $10) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQA0HsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IGZhbHNlCn10AARleHBydAHbewogICJvcCI6IHsKICAgICJuYW1lIjogIi0iLAogICAgImtpbmQiOiAiTUlOVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICIrIiwKICAgICAgICAia2luZCI6ICJQTFVTIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDEsCiAgICAgICAgICAibmFtZSI6ICIkMSIKICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJpbnB1dCI6IDAsCiAgICAgICJuYW1lIjogIiQwIgogICAgfQogIF0sCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAnQAB2JhbGFuY2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05HdAADYWdlfnEAfgAKdAAHSU5URUdFUnh4\"}","lang":"opensearch_compounded_script","params":{"NULL_DIRECTION":"FIRST","DIRECTION":"ASCENDING","utcTimestamp": 0}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[-(+($10, $7), $10) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCsnsKICAib3AiOiB7CiAgICAibmFtZSI6ICItIiwKICAgICJraW5kIjogIk1JTlVTIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiKyIsCiAgICAgICAgImtpbmQiOiAiUExVUyIsCiAgICAgICAgInN5bnRheCI6ICJCSU5BUlkiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0sCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0,0],"DIGESTS":["age","balance","age"]}},"type":"number","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml index c54440bbf61..07a4d5e2268 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml @@ -5,4 +5,4 @@ calcite: LogicalProject($f1=[+($3, 1)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"cnt":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAfnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AQF7CiAgIm9wIjogewogICAgIm5hbWUiOiAiKyIsCiAgICAia2luZCI6ICJQTFVTIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiAwLAogICAgICAibmFtZSI6ICIkMCIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQAB2JhbGFuY2V+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05HeHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"cnt":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBRXsKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",1]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_eval_min.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_eval_min.yaml index 71e2911eb09..d28a4ef30ec 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_eval_min.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_eval_min.yaml @@ -5,4 +5,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..10=[{inputs}], expr#11=[1], expr#12=[2], expr#13=[3], expr#14=['banana':VARCHAR], expr#15=[SCALAR_MIN($t11, $t12, $t13, $t8, $t14)], proj#0..10=[{exprs}], $f11=[$t15]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_function_script_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_function_script_push.yaml index 7d61dbc0b8b..8c4b080c8f2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_function_script_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_function_script_push.yaml @@ -5,4 +5,4 @@ calcite: LogicalFilter(condition=[AND(=(CHAR_LENGTH($1), 5), =(ABS($8), 32), =($3, 39225))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, balance, age], SCRIPT->AND(=(CHAR_LENGTH($0), 5), =(ABS($2), 32), =($1, 39225)), PROJECT->[firstname, age], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAmHsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJmaXJzdG5hbWUiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQBrXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNIQVJfTEVOR1RIIiwKICAgICAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogNSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgALeHB+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4AEXQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABh4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+ABoAAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+AAx+cQB+ABB0AAZTVFJJTkd+cQB+ABR0AAdLZXl3b3JkcQB+ABl4eHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAensKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQBqXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkFCUyIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgIm5hbWUiOiAiJDAiCiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDMyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3h4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},{"term":{"balance":{"value":39225,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, balance, age], SCRIPT->AND(=(CHAR_LENGTH($0), 5), =(ABS($2), 32), =($1, 39225)), PROJECT->[firstname, age], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCG3sKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNIQVJfTEVOR1RIIiwKICAgICAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["firstname.keyword",5]}},"boost":1.0}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQB+HsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkFCUyIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["age",32]}},"boost":1.0}},{"term":{"balance":{"value":39225,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_ip_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_ip_push.json deleted file mode 100644 index bb2cf27c0a5..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_ip_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(host=[$FIELD_INDEX])\n LogicalFilter(condition=[CIDRMATCH($FIELD_INDEX, '0.0.0.0/24':VARCHAR)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[PROJECT->[host], SCRIPT->CIDRMATCH($FIELD_INDEX, '0.0.0.0/24':VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAknsKICAiZmllbGRzIjogWwogICAgewogICAgICAidWR0IjogIkVYUFJfSVAiLAogICAgICAidHlwZSI6ICJPVEhFUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogImhvc3QiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQB9XsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSURSTUFUQ0giLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICIwLjAuMC4wLzI0IiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdLAogICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIkJPT0xFQU4iLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQABGhvc3R+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAJJUHh4\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},\"_source\":{\"includes\":[\"host\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_ip_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_ip_push.yaml new file mode 100644 index 00000000000..7b9bb6bc9e6 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_ip_push.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(host=[$FIELD_INDEX]) + LogicalFilter(condition=[CIDRMATCH($FIELD_INDEX, '0.0.0.0/24':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[PROJECT->[host], SCRIPT->CIDRMATCH($FIELD_INDEX, '0.0.0.0/24':VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCR3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSURSTUFUQ0giLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ1ZHQiOiAiRVhQUl9JUCIsCiAgICAgICAgInR5cGUiOiAiT1RIRVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9LAogICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAiZHluYW1pYyI6IGZhbHNlCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["host","0.0.0.0/24"]}},"boost":1.0}},"_source":{"includes":["host"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml index 4992957b230..26be3b4c787 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_push.yaml @@ -5,4 +5,4 @@ calcite: LogicalFilter(condition=[AND(=($1, 'Amber'), =(-($8, 2), 30))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, age], SCRIPT->AND(=($0, 'Amber'), =(-($1, 2), 30)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"term":{"firstname.keyword":{"value":"Amber","boost":1.0}}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAensKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQCcnsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIi0iLAogICAgICAgICJraW5kIjogIk1JTlVTIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogMiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDMwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3h4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, age], SCRIPT->AND(=($0, 'Amber'), =(-($1, 2), 30)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"bool":{"must":[{"term":{"firstname.keyword":{"value":"Amber","boost":1.0}}},{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCxnsKICAib3AiOiB7CiAgICAibmFtZSI6ICI9IiwKICAgICJraW5kIjogIkVRVUFMUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIi0iLAogICAgICAgICJraW5kIjogIk1JTlVTIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["age",2,30]}},"boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["firstname","age"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_isblank.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_isblank.yaml index aa739610250..2877a74bf7b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_isblank.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_isblank.yaml @@ -5,4 +5,4 @@ calcite: LogicalFilter(condition=[OR(IS NULL($1), IS EMPTY(TRIM(FLAG(BOTH), ' ', $1)))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->OR(IS NULL($1), IS EMPTY(TRIM(FLAG(BOTH), ' ', $1))), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAmHsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJmaXJzdG5hbWUiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQENHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJPUiIsCiAgICAia2luZCI6ICJPUiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIklTIE5VTEwiLAogICAgICAgICJraW5kIjogIklTX05VTEwiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJJUyBFTVBUWSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVIiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiVFJJTSIsCiAgICAgICAgICAgICJraW5kIjogIlRSSU0iLAogICAgICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICAgICAgfSwKICAgICAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICAgICAgewogICAgICAgICAgICAgICJsaXRlcmFsIjogIkJPVEgiLAogICAgICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAiU1lNQk9MIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImxpdGVyYWwiOiAiICIsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAgICAgInByZWNpc2lvbiI6IDEKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgICAgICB9CiAgICAgICAgICBdCiAgICAgICAgfQogICAgICBdCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAAJZmlyc3RuYW1lc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4ADH5xAH4AEHQABlNUUklOR35xAH4AFHQAB0tleXdvcmRxAH4AGXh4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->OR(IS NULL($1), IS EMPTY(TRIM(FLAG(BOTH), ' ', $1))), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQFGXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJPUiIsCiAgICAia2luZCI6ICJPUiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIklTIE5VTEwiLAogICAgICAgICJraW5kIjogIklTX05VTEwiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJJUyBFTVBUWSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVIiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJvcCI6IHsKICAgICAgICAgICAgIm5hbWUiOiAiVFJJTSIsCiAgICAgICAgICAgICJraW5kIjogIlRSSU0iLAogICAgICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICAgICAgfSwKICAgICAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICAgICAgewogICAgICAgICAgICAgICJsaXRlcmFsIjogIkJPVEgiLAogICAgICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAgICAgInR5cGUiOiAiU1lNQk9MIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgICAgICAgICAgfQogICAgICAgICAgICB9LAogICAgICAgICAgICB7CiAgICAgICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAgICAgInByZWNpc2lvbiI6IDEKICAgICAgICAgICAgICB9CiAgICAgICAgICAgIH0sCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgICAgIH0KICAgICAgICAgICAgfQogICAgICAgICAgXQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,0],"DIGESTS":["firstname.keyword"," ","firstname.keyword"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty.yaml index 349be65b454..df97332ae98 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty.yaml @@ -5,4 +5,4 @@ calcite: LogicalFilter(condition=[OR(IS NULL($1), IS EMPTY($1))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->OR(IS NULL($1), IS EMPTY($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAmHsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJmaXJzdG5hbWUiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQCBXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJPUiIsCiAgICAia2luZCI6ICJPUiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIklTIE5VTEwiLAogICAgICAgICJraW5kIjogIklTX05VTEwiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJJUyBFTVBUWSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVIiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAlmaXJzdG5hbWVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AC3hwfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+ABF0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAYeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAaAAAAAHNxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgAMfnEAfgAQdAAGU1RSSU5HfnEAfgAUdAAHS2V5d29yZHEAfgAZeHh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->OR(IS NULL($1), IS EMPTY($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQC13sKICAib3AiOiB7CiAgICAibmFtZSI6ICJPUiIsCiAgICAia2luZCI6ICJPUiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIklTIE5VTEwiLAogICAgICAgICJraW5kIjogIklTX05VTEwiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJJUyBFTVBUWSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVIiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["firstname.keyword","firstname.keyword"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty_or_others.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty_or_others.yaml index b940a11198e..b3d22c2bfa1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty_or_others.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_isempty_or_others.yaml @@ -5,4 +5,4 @@ calcite: LogicalFilter(condition=[OR(=($4, 'M'), IS NULL($1), IS EMPTY($1))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->OR(IS NULL($1), =($4, 'M'), IS EMPTY($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQBBHsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJmaXJzdG5hbWUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJnZW5kZXIiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQDdXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJPUiIsCiAgICAia2luZCI6ICJPUiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIklTIE5VTEwiLAogICAgICAgICJraW5kIjogIklTX05VTEwiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICI9IiwKICAgICAgICAia2luZCI6ICJFUVVBTFMiLAogICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMSwKICAgICAgICAgICJuYW1lIjogIiQxIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiTSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfSwKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIklTIEVNUFRZIiwKICAgICAgICAia2luZCI6ICJPVEhFUiIsCiAgICAgICAgInN5bnRheCI6ICJQT1NURklYIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAnQACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgALeHB+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4AEXQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABh4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+ABoAAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+AAx+cQB+ABB0AAZTVFJJTkd+cQB+ABR0AAdLZXl3b3JkcQB+ABl4dAAGZ2VuZGVyc3EAfgAKcQB+ABJxAH4AFXEAfgAZc3EAfgAAAAAAA3cEAAAAAnEAfgAecQB+AB94eHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->OR(IS NULL($1), =($4, 'M'), IS EMPTY($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQEs3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJPUiIsCiAgICAia2luZCI6ICJPUiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIklTIE5VTEwiLAogICAgICAgICJraW5kIjogIklTX05VTEwiLAogICAgICAgICJzeW50YXgiOiAiUE9TVEZJWCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICI9IiwKICAgICAgICAia2luZCI6ICJFUVVBTFMiLAogICAgICAgICJzeW50YXgiOiAiQklOQVJZIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfSwKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIklTIEVNUFRZIiwKICAgICAgICAia2luZCI6ICJPVEhFUiIsCiAgICAgICAgInN5bnRheCI6ICJQT1NURklYIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDMsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,0,2,0],"DIGESTS":["firstname.keyword","gender.keyword","M","firstname.keyword"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_min_max_agg_on_derived_field.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_min_max_agg_on_derived_field.yaml index 805c42527a8..f083def95a4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_min_max_agg_on_derived_field.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_min_max_agg_on_derived_field.yaml @@ -5,4 +5,4 @@ calcite: LogicalProject(balance2=[CEIL(DIVIDE($3, 10000.0:DECIMAL(6, 1)))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},MIN(balance2)=MIN($0),MAX(balance2)=MAX($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"MIN(balance2)":{"min":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAfnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0Awl7CiAgIm9wIjogewogICAgIm5hbWUiOiAiQ0VJTCIsCiAgICAia2luZCI6ICJDRUlMIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJESVZJREUiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAxMDAwMC4wLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkRFQ0lNQUwiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IDYsCiAgICAgICAgICAgICJzY2FsZSI6IDEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiREVDSU1BTCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogMjcsCiAgICAgICAgInNjYWxlIjogNwogICAgICB9LAogICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICJkeW5hbWljIjogZmFsc2UKICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAdiYWxhbmNlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3h4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}},"MAX(balance2)":{"max":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAfnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0Awl7CiAgIm9wIjogewogICAgIm5hbWUiOiAiQ0VJTCIsCiAgICAia2luZCI6ICJDRUlMIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJESVZJREUiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAxMDAwMC4wLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIkRFQ0lNQUwiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IDYsCiAgICAgICAgICAgICJzY2FsZSI6IDEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiREVDSU1BTCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogMjcsCiAgICAgICAgInNjYWxlIjogNwogICAgICB9LAogICAgICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgICAgICJkeW5hbWljIjogZmFsc2UKICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAdiYWxhbmNlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3h4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},MIN(balance2)=MIN($0),MAX(balance2)=MAX($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"MIN(balance2)":{"min":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQDVHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDRUlMIiwKICAgICJraW5kIjogIkNFSUwiLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkRJVklERSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAibGl0ZXJhbCI6IDEwMDAwLjAsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiREVDSU1BTCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogNiwKICAgICAgICAgICAgInNjYWxlIjogMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJERUNJTUFMIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAyNywKICAgICAgICAic2NhbGUiOiA3CiAgICAgIH0sCiAgICAgICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAgICAgImR5bmFtaWMiOiBmYWxzZQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["balance"]}}}},"MAX(balance2)":{"max":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQDVHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDRUlMIiwKICAgICJraW5kIjogIkNFSUwiLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkRJVklERSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAibGl0ZXJhbCI6IDEwMDAwLjAsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiREVDSU1BTCIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogNiwKICAgICAgICAgICAgInNjYWxlIjogMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJERUNJTUFMIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICJwcmVjaXNpb24iOiAyNywKICAgICAgICAic2NhbGUiOiA3CiAgICAgIH0sCiAgICAgICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAgICAgImR5bmFtaWMiOiBmYWxzZQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["balance"]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml index 16c87b3ae02..dfcb6e6b7e0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..2=[{inputs}], expr#3=[PATTERN_PARSER($t0, $t2)], expr#4=['pattern'], expr#5=[ITEM($t3, $t4)], expr#6=[SAFE_CAST($t5)], expr#7=['tokens'], expr#8=[ITEM($t3, $t7)], expr#9=[SAFE_CAST($t8)], patterns_field=[$t6], pattern_count=[$t1], tokens=[$t9], sample_logs=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},pattern_count=COUNT($1),sample_logs=TAKE($0, $2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"patterns_field":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlHsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IGZhbHNlCn10AARleHBydAYZewogICJvcCI6IHsKICAgICJuYW1lIjogIkNBU0UiLAogICAgImtpbmQiOiAiQ0FTRSIsCiAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJTRUFSQ0giLAogICAgICAgICJraW5kIjogIlNFQVJDSCIsCiAgICAgICAgInN5bnRheCI6ICJJTlRFUk5BTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogewogICAgICAgICAgICAicmFuZ2VTZXQiOiBbCiAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgInNpbmdsZXRvbiIsCiAgICAgICAgICAgICAgICAie1widmFsdWVcIjpcIlwiLFwiY2hhcnNldE5hbWVcIjpcIklTTy04ODU5LTFcIixcImNvbGxhdGlvblwiOntcImNvbGxhdGlvbk5hbWVcIjpcIklTTy04ODU5LTEkZW5fVVMkcHJpbWFyeVwiLFwiY29lcmNpYmlsaXR5XCI6XCJJTVBMSUNJVFwiLFwibG9jYWxlXCI6XCJlbl9VU1wifSxcImNoYXJzZXRcIjpcIklTTy04ODU5LTFcIixcInZhbHVlQnl0ZXNcIjpudWxsfSIKICAgICAgICAgICAgICBdCiAgICAgICAgICAgIF0sCiAgICAgICAgICAgICJudWxsQXMiOiAiVFJVRSIKICAgICAgICAgIH0sCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIiIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiUkVHRVhQX1JFUExBQ0UiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiW2EtekEtWjAtOV0rIiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiPCo+IiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQABWVtYWlsc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4ADH5xAH4AEHQABlNUUklOR35xAH4AFHQAB0tleXdvcmRxAH4AGXh4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"pattern_count":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlHsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IGZhbHNlCn10AARleHBydAYZewogICJvcCI6IHsKICAgICJuYW1lIjogIkNBU0UiLAogICAgImtpbmQiOiAiQ0FTRSIsCiAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJTRUFSQ0giLAogICAgICAgICJraW5kIjogIlNFQVJDSCIsCiAgICAgICAgInN5bnRheCI6ICJJTlRFUk5BTCIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogewogICAgICAgICAgICAicmFuZ2VTZXQiOiBbCiAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgInNpbmdsZXRvbiIsCiAgICAgICAgICAgICAgICAie1widmFsdWVcIjpcIlwiLFwiY2hhcnNldE5hbWVcIjpcIklTTy04ODU5LTFcIixcImNvbGxhdGlvblwiOntcImNvbGxhdGlvbk5hbWVcIjpcIklTTy04ODU5LTEkZW5fVVMkcHJpbWFyeVwiLFwiY29lcmNpYmlsaXR5XCI6XCJJTVBMSUNJVFwiLFwibG9jYWxlXCI6XCJlbl9VU1wifSxcImNoYXJzZXRcIjpcIklTTy04ODU5LTFcIixcInZhbHVlQnl0ZXNcIjpudWxsfSIKICAgICAgICAgICAgICBdCiAgICAgICAgICAgIF0sCiAgICAgICAgICAgICJudWxsQXMiOiAiVFJVRSIKICAgICAgICAgIH0sCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIiIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiUkVHRVhQX1JFUExBQ0UiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiW2EtekEtWjAtOV0rIiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiPCo+IiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQABWVtYWlsc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4ADH5xAH4AEHQABlNUUklOR35xAH4AFHQAB0tleXdvcmRxAH4AGXh4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}},"sample_logs":{"top_hits":{"from":0,"size":10,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"email.keyword"}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},pattern_count=COUNT($1),sample_logs=TAKE($0, $2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"patterns_field":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQG6HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0VBUkNIIiwKICAgICAgICAia2luZCI6ICJTRUFSQ0giLAogICAgICAgICJzeW50YXgiOiAiSU5URVJOQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAibGl0ZXJhbCI6IHsKICAgICAgICAgICAgInJhbmdlU2V0IjogWwogICAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgICJzaW5nbGV0b24iLAogICAgICAgICAgICAgICAgIntcInZhbHVlXCI6XCJcIixcImNoYXJzZXROYW1lXCI6XCJJU08tODg1OS0xXCIsXCJjb2xsYXRpb25cIjp7XCJjb2xsYXRpb25OYW1lXCI6XCJJU08tODg1OS0xJGVuX1VTJHByaW1hcnlcIixcImNvZXJjaWJpbGl0eVwiOlwiSU1QTElDSVRcIixcImxvY2FsZVwiOlwiZW5fVVNcIn0sXCJjaGFyc2V0XCI6XCJJU08tODg1OS0xXCIsXCJ2YWx1ZUJ5dGVzXCI6bnVsbH0iCiAgICAgICAgICAgICAgXQogICAgICAgICAgICBdLAogICAgICAgICAgICAibnVsbEFzIjogIlRSVUUiCiAgICAgICAgICB9LAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMywKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDQsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,0,2,2],"DIGESTS":["email.keyword","","email.keyword","[a-zA-Z0-9]+","<*>"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"pattern_count":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQG6HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0VBUkNIIiwKICAgICAgICAia2luZCI6ICJTRUFSQ0giLAogICAgICAgICJzeW50YXgiOiAiSU5URVJOQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAibGl0ZXJhbCI6IHsKICAgICAgICAgICAgInJhbmdlU2V0IjogWwogICAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgICJzaW5nbGV0b24iLAogICAgICAgICAgICAgICAgIntcInZhbHVlXCI6XCJcIixcImNoYXJzZXROYW1lXCI6XCJJU08tODg1OS0xXCIsXCJjb2xsYXRpb25cIjp7XCJjb2xsYXRpb25OYW1lXCI6XCJJU08tODg1OS0xJGVuX1VTJHByaW1hcnlcIixcImNvZXJjaWJpbGl0eVwiOlwiSU1QTElDSVRcIixcImxvY2FsZVwiOlwiZW5fVVNcIn0sXCJjaGFyc2V0XCI6XCJJU08tODg1OS0xXCIsXCJ2YWx1ZUJ5dGVzXCI6bnVsbH0iCiAgICAgICAgICAgICAgXQogICAgICAgICAgICBdLAogICAgICAgICAgICAibnVsbEFzIjogIlRSVUUiCiAgICAgICAgICB9LAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMywKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDQsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,0,2,2],"DIGESTS":["email.keyword","","email.keyword","[a-zA-Z0-9]+","<*>"]}}}},"sample_logs":{"top_hits":{"from":0,"size":10,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"email.keyword"}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_regex.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_regex.yaml index a1c7349ac3f..7ac431a8b1d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_regex.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_regex.yaml @@ -6,4 +6,4 @@ calcite: LogicalFilter(condition=[REGEXP_CONTAINS($10, '^[A-Z][a-z]+$':VARCHAR)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->REGEXP_CONTAINS($10, '^[A-Z][a-z]+$':VARCHAR), LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAl3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJsYXN0bmFtZSIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IGZhbHNlCn10AARleHBydAFCewogICJvcCI6IHsKICAgICJuYW1lIjogIlJFR0VYUF9DT05UQUlOUyIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiAwLAogICAgICAibmFtZSI6ICIkMCIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIl5bQS1aXVthLXpdKyQiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQACGxhc3RuYW1lc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4ADH5xAH4AEHQABlNUUklOR35xAH4AFHQAB0tleXdvcmRxAH4AGXh4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->REGEXP_CONTAINS($10, '^[A-Z][a-z]+$':VARCHAR), LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBknsKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfQ09OVEFJTlMiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["lastname.keyword","^[A-Z][a-z]+$"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_regex_negated.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_regex_negated.yaml index e94f8ab11c3..0dfaa8b267e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_regex_negated.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_regex_negated.yaml @@ -6,4 +6,4 @@ calcite: LogicalFilter(condition=[NOT(REGEXP_CONTAINS($10, '.*son$':VARCHAR))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->NOT(REGEXP_CONTAINS($10, '.*son$':VARCHAR)), LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAl3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJsYXN0bmFtZSIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IGZhbHNlCn10AARleHBydAHzewogICJvcCI6IHsKICAgICJuYW1lIjogIk5PVCIsCiAgICAia2luZCI6ICJOT1QiLAogICAgInN5bnRheCI6ICJQUkVGSVgiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJSRUdFWFBfQ09OVEFJTlMiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMCwKICAgICAgICAgICJuYW1lIjogIiQwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAiLipzb24kIiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQACGxhc3RuYW1lc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4ADH5xAH4AEHQABlNUUklOR35xAH4AFHQAB0tleXdvcmRxAH4AGXh4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->NOT(REGEXP_CONTAINS($10, '.*son$':VARCHAR)), LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCWnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJOT1QiLAogICAgImtpbmQiOiAiTk9UIiwKICAgICJzeW50YXgiOiAiUFJFRklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiUkVHRVhQX0NPTlRBSU5TIiwKICAgICAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["lastname.keyword",".*son$"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_regexp_match_in_where.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_regexp_match_in_where.json deleted file mode 100644 index ae0ce763383..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_regexp_match_in_where.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(name=[$0])\n LogicalFilter(condition=[REGEXP_CONTAINS($0, 'hello':VARCHAR)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_strings]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_strings]], PushDownContext=[[PROJECT->[name], SCRIPT->REGEXP_CONTAINS($0, 'hello':VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAk3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJuYW1lIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0ATp7CiAgIm9wIjogewogICAgIm5hbWUiOiAiUkVHRVhQX0NPTlRBSU5TIiwKICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDAsCiAgICAgICJuYW1lIjogIiQwIgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAiaGVsbG8iLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQABG5hbWVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AC3hwfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+ABF0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAYeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAaAAAAAHNxAH4AAAAAAAN3BAAAAAB4eHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},\"_source\":{\"includes\":[\"name\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_regexp_match_in_where.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_regexp_match_in_where.yaml new file mode 100644 index 00000000000..ace954902a4 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_regexp_match_in_where.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(name=[$0]) + LogicalFilter(condition=[REGEXP_CONTAINS($0, 'hello':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_strings]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_strings]], PushDownContext=[[PROJECT->[name], SCRIPT->REGEXP_CONTAINS($0, 'hello':VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBknsKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfQ09OVEFJTlMiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2],"DIGESTS":["name","hello"]}},"boost":1.0}},"_source":{"includes":["name"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.yaml index 87f2c9ab813..a5f43d257ce 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.yaml @@ -7,4 +7,4 @@ calcite: LogicalFilter(condition=[>(CHAR_LENGTH($2), 0)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[address], SCRIPT->>(CHAR_LENGTH($0), 0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), address_length], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJhZGRyZXNzIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiPiIsCiAgICAia2luZCI6ICJHUkVBVEVSX1RIQU4iLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAgICAgImtpbmQiOiAiQ0hBUl9MRU5HVEgiLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgIm5hbWUiOiAiJDAiCiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAdhZGRyZXNzc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAAAeHh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"_source":{"includes":["address"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"address_length":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJhZGRyZXNzIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AKZ7CiAgIm9wIjogewogICAgIm5hbWUiOiAiQ0hBUl9MRU5HVEgiLAogICAgImtpbmQiOiAiQ0hBUl9MRU5HVEgiLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAAHYWRkcmVzc3NyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgALeHB+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4AEXQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABh4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+ABoAAAAAc3EAfgAAAAAAA3cEAAAAAHh4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"value_type":"long","missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[address], SCRIPT->>(CHAR_LENGTH($0), 0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), address_length], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCIXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI+IiwKICAgICJraW5kIjogIkdSRUFURVJfVEhBTiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNIQVJfTEVOR1RIIiwKICAgICAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2],"DIGESTS":["address",0]}},"boost":1.0}},"_source":{"includes":["address"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"address_length":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1],"DIGESTS":["address"]}},"missing_bucket":true,"value_type":"long","missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json index 5844a594d72..0bc349d23dd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json @@ -1,7 +1,7 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(firstname=[$1], age=[$8], address=[$2])\n LogicalFilter(condition=[AND(=($2, '671 Bristol Street'), =(-($8, 2), 30))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, address, age], SCRIPT->AND(=($1, '671 Bristol Street'), =(-($2, 2), 30)), PROJECT->[firstname, age, address], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"must\":[{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"EQUALS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"input\\\\\\\": 0,\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"$0\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"literal\\\\\\\": \\\\\\\"671 Bristol Street\\\\\\\",\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": false,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n }\\\\n ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"EQUALS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"-\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"MINUS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"input\\\\\\\": 0,\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"$0\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"literal\\\\\\\": 2,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"INTEGER\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": false\\\\n }\\\\n }\\\\n ],\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"literal\\\\\\\": 30,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"INTEGER\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": false\\\\n }\\\\n }\\\\n ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"firstname\",\"age\",\"address\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n", + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, address, age], SCRIPT->AND(=($1, '671 Bristol Street'), =(-($2, 2), 30)), PROJECT->[firstname, age, address], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"must\":[{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"EQUALS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 0,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 1,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": false,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n }\\\\n ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*,\"SOURCES\":[1,2],\"DIGESTS\":[\"address\",\"671 Bristol Street\"]}},\"boost\":1.0}},{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"EQUALS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"-\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"MINUS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 0,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 1,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"INTEGER\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": false\\\\n }\\\\n }\\\\n ],\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"dynamicParam\\\\\\\": 2,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"INTEGER\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": false\\\\n }\\\\n }\\\\n ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*,\"SOURCES\":[0,2,2],\"DIGESTS\":[\"age\",2,30]}},\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"firstname\",\"age\",\"address\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n", "extended": "public org.apache.calcite.linq4j.Enumerable bind(final org.apache.calcite.DataContext root) {\n final org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan v1stashed = (org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan) root.get(\"v1stashed\");\n return v1stashed.scan();\n}\n\n\npublic Class getElementType() {\n return java.lang.Object[].class;\n}\n\n\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml index ee2254f5420..faaaa2018ef 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_complex_and_simple_expr.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..12=[{inputs}], expr#13=[+($t10, $t7)], expr#14=[1], expr#15=[+($t7, $t14)], proj#0..13=[{exprs}], $f14=[$t15]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[+($10, $7) ASCENDING NULLS_FIRST, balance ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQA0HsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IGZhbHNlCn10AARleHBydADFewogICJvcCI6IHsKICAgICJuYW1lIjogIisiLAogICAgImtpbmQiOiAiUExVUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9LAogICAgewogICAgICAiaW5wdXQiOiAxLAogICAgICAibmFtZSI6ICIkMSIKICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAJ0AAdiYWxhbmNlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QAA2FnZX5xAH4ACnQAB0lOVEVHRVJ4eA==\"}","lang":"opensearch_compounded_script","params":{"NULL_DIRECTION":"FIRST","DIRECTION":"ASCENDING","utcTimestamp": 0}},"type":"number","order":"asc"}},{"balance":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[+($10, $7) ASCENDING NULLS_FIRST, balance ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBRHsKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,0],"DIGESTS":["age","balance"]}},"type":"number","order":"asc"}},{"balance":{"order":"asc","missing":"_first"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_type_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_type_push.json index 054bc203640..c9959ce01da 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_type_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_type_push.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8])\n LogicalSort(sort0=[$17], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], $f17=[SAFE_CAST($8)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], SORT_EXPR->[SAFE_CAST($0) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAensKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQA3XsKICAib3AiOiB7CiAgICAibmFtZSI6ICJTQUZFX0NBU1QiLAogICAgImtpbmQiOiAiU0FGRV9DQVNUIiwKICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9CiAgXSwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIkRPVUJMRSIsCiAgICAibnVsbGFibGUiOiB0cnVlCiAgfQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAAAx3CAAAABAAAAABdAADYWdlfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3h4\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"NULL_DIRECTION\":\"FIRST\",\"DIRECTION\":\"ASCENDING\",\"utcTimestamp\":*}},\"type\":\"number\",\"order\":\"asc\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], SORT_EXPR->[SAFE_CAST($0) ASCENDING NULLS_FIRST], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXQBHHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJTQUZFX0NBU1QiLAogICAgImtpbmQiOiAiU0FGRV9DQVNUIiwKICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9CiAgXSwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIkRPVUJMRSIsCiAgICAibnVsbGFibGUiOiB0cnVlCiAgfQp9\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"MISSING_MAX\":false,\"utcTimestamp\":*,\"SOURCES\":[0],\"DIGESTS\":[\"age\"]}},\"type\":\"number\",\"order\":\"asc\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function.yaml index c3f7ffe2a93..9eb6e726fbf 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_text_like_function.yaml @@ -5,4 +5,4 @@ calcite: LogicalFilter(condition=[ILIKE($2, '%Holmes%', '\')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->ILIKE($2, '%Holmes%', '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJhZGRyZXNzIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0Aap7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSUxJS0UiLAogICAgImtpbmQiOiAiTElLRSIsCiAgICAic3ludGF4IjogIlNQRUNJQUwiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDAsCiAgICAgICJuYW1lIjogIiQwIgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAiJUhvbG1lcyUiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogOAogICAgICB9CiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICJcXCIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAxCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAdhZGRyZXNzc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAAAeHh4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], SCRIPT->ILIKE($2, '%Holmes%', '\'), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCAXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJJTElLRSIsCiAgICAia2luZCI6ICJMSUtFIiwKICAgICJzeW50YXgiOiAiU1BFQ0lBTCIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogOAogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IDEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2,2],"DIGESTS":["address","%Holmes%","\\"]}},"boost":1.0}},"_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/udf_geoip_in_agg_pushed.yaml b/integ-test/src/test/resources/expectedOutput/calcite/udf_geoip_in_agg_pushed.yaml index 30033cb8bff..34bb5977180 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/udf_geoip_in_agg_pushed.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/udf_geoip_in_agg_pushed.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(info.city=[ITEM(GEOIP('my-datasource':VARCHAR, $3), 'city')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), info.city], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"info.city":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAknsKICAiZmllbGRzIjogWwogICAgewogICAgICAidWR0IjogIkVYUFJfSVAiLAogICAgICAidHlwZSI6ICJPVEhFUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogImhvc3QiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQETnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJJVEVNIiwKICAgICJraW5kIjogIklURU0iLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiR0VPSVAiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAibXktZGF0YXNvdXJjZSIsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDAsCiAgICAgICAgICAibmFtZSI6ICIkMCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiTUFQIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAia2V5IjogewogICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgIH0sCiAgICAgICAgInZhbHVlIjogewogICAgICAgICAgInR5cGUiOiAiQU5ZIiwKICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAgICAgInNjYWxlIjogLTIxNDc0ODM2NDgKICAgICAgICB9CiAgICAgIH0sCiAgICAgICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAgICAgImR5bmFtaWMiOiBmYWxzZQogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAiY2l0eSIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiA0CiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AARob3N0fnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAACSVB4eA==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), info.city], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"info.city":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQErXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJJVEVNIiwKICAgICJraW5kIjogIklURU0iLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiR0VPSVAiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ1ZHQiOiAiRVhQUl9JUCIsCiAgICAgICAgICAgICJ0eXBlIjogIk9USEVSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJNQVAiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJrZXkiOiB7CiAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgfSwKICAgICAgICAidmFsdWUiOiB7CiAgICAgICAgICAidHlwZSI6ICJBTlkiLAogICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICAgICAic2NhbGUiOiAtMjE0NzQ4MzY0OAogICAgICAgIH0KICAgICAgfSwKICAgICAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IDQKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[2,0,2],"DIGESTS":["my-datasource","host","city"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eval_min.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eval_min.yaml index e7c0af846f0..6468a521e38 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eval_min.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_eval_min.yaml @@ -6,4 +6,4 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..16=[{inputs}], expr#17=[1], expr#18=[2], expr#19=[3], expr#20=['banana':VARCHAR], expr#21=[SCALAR_MIN($t17, $t18, $t19, $t8, $t20)], proj#0..10=[{exprs}], new=[$t21]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_ip_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_ip_push.json deleted file mode 100644 index 545630e2166..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_ip_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(host=[$0])\n LogicalFilter(condition=[CIDRMATCH($0, '0.0.0.0/24':VARCHAR)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..11=[{inputs}], expr#12=['0.0.0.0/24':VARCHAR], expr#13=[CIDRMATCH($t0, $t12)], host=[$t0], $condition=[$t13])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_ip_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_ip_push.yaml new file mode 100644 index 00000000000..977d61c0e16 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_ip_push.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(host=[$FIELD_INDEX]) + LogicalFilter(condition=[CIDRMATCH($FIELD_INDEX, '0.0.0.0/24':VARCHAR)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..11=[{inputs}], expr#12=['0.0.0.0/24':VARCHAR], expr#13=[CIDRMATCH($FIELD_INDEX, $t12)], host=[$FIELD_INDEX], $condition=[$t13]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_agg_with_sum_enhancement.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_agg_with_sum_enhancement.json deleted file mode 100644 index e965b5b5dc2..00000000000 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_agg_with_sum_enhancement.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "root": { - "name": "ProjectOperator", - "description": { - "fields": "[sum(balance), sum(balance + 100), sum(balance - 100), sum(balance * 100), sum(balance / 100), gender]" - }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_bank, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"sum(balance)\":{\"sum\":{\"field\":\"balance\"}},\"sum(balance + 100)\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAdiYWxhbmNlc3IAGmphdmEudXRpbC5BcnJheXMkQXJyYXlMaXN02aQ8vs2IBtICAAFbAAFhdAATW0xqYXZhL2xhbmcvT2JqZWN0O3hwdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXEAfgANcQB+AA1+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05Hc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAGR4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDR0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGiQdxM68QzgeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUExxAH4AFQ==\\\"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(balance - 100)\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAdiYWxhbmNlc3IAGmphdmEudXRpbC5BcnJheXMkQXJyYXlMaXN02aQ8vs2IBtICAAFbAAFhdAATW0xqYXZhL2xhbmcvT2JqZWN0O3hwdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXEAfgANcQB+AA1+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05Hc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAGR4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAS1xAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAHmxhbWJkYSRzdWJ0cmFjdEJhc2UkN2Q5MGFjMDIkNHQAfShMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADV2cgAyb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25EU0wAAAAAAAAAAAAAAHhwcQB+ADBxAH4AMXEAfgAydAAyb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25EU0x0ACVsYW1iZGEkbnVsbE1pc3NpbmdIYW5kbGluZyRhNTAwNTI4MSQxdAC8KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4ANXEAfgA3dAA+b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlVHJpRnVuY3Rpb25xAH4AMXQASihMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7cQB+ADh0ABZsYW1iZGEkaW1wbCRhMGZiMzRkNCQxdAD3KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3QAuChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AI3NyADlvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvblByb3BlcnRpZXPPPFljm6j5swIAA0wADWN1cnJlbnRab25lSWR0ABJMamF2YS90aW1lL1pvbmVJZDtMAApub3dJbnN0YW50dAATTGphdmEvdGltZS9JbnN0YW50O0wACXF1ZXJ5VHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhlY3V0b3IvUXVlcnlUeXBlO3hwc3IADWphdmEudGltZS5TZXKVXYS6GyJIsgwAAHhwdwIIAHhzcQB+AEV3DQIAAAAAaJB3EzrxDOB4fnIAJW9yZy5vcGVuc2VhcmNoLnNxbC5leGVjdXRvci5RdWVyeVR5cGUAAAAAAAAAABIAAHhxAH4AFHQAA1BQTHEAfgAV\\\"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(balance * 100)\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAdiYWxhbmNlc3IAGmphdmEudXRpbC5BcnJheXMkQXJyYXlMaXN02aQ8vs2IBtICAAFbAAFhdAATW0xqYXZhL2xhbmcvT2JqZWN0O3hwdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXEAfgANcQB+AA1+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05Hc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAGR4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAASpxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAHmxhbWJkYSRtdWx0aXBseUJhc2UkN2Q5MGFjMDIkNHQAfShMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADV2cgAyb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25EU0wAAAAAAAAAAAAAAHhwcQB+ADBxAH4AMXEAfgAydAAyb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25EU0x0ACVsYW1iZGEkbnVsbE1pc3NpbmdIYW5kbGluZyRhNTAwNTI4MSQxdAC8KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4ANXEAfgA3dAA+b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlVHJpRnVuY3Rpb25xAH4AMXQASihMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7cQB+ADh0ABZsYW1iZGEkaW1wbCRhMGZiMzRkNCQxdAD3KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3QAuChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AI3NyADlvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvblByb3BlcnRpZXPPPFljm6j5swIAA0wADWN1cnJlbnRab25lSWR0ABJMamF2YS90aW1lL1pvbmVJZDtMAApub3dJbnN0YW50dAATTGphdmEvdGltZS9JbnN0YW50O0wACXF1ZXJ5VHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhlY3V0b3IvUXVlcnlUeXBlO3hwc3IADWphdmEudGltZS5TZXKVXYS6GyJIsgwAAHhwdwIIAHhzcQB+AEV3DQIAAAAAaJB3EzrxDOB4fnIAJW9yZy5vcGVuc2VhcmNoLnNxbC5leGVjdXRvci5RdWVyeVR5cGUAAAAAAAAAABIAAHhxAH4AFHQAA1BQTHEAfgAV\\\"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(balance / 100)\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAdiYWxhbmNlc3IAGmphdmEudXRpbC5BcnJheXMkQXJyYXlMaXN02aQ8vs2IBtICAAFbAAFhdAATW0xqYXZhL2xhbmcvT2JqZWN0O3hwdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXEAfgANcQB+AA1+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05Hc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAGR4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAS9xAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAHGxhbWJkYSRkaXZpZGVCYXNlJDdkOTBhYzAyJDR0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGiQdxM68QzgeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUExxAH4AFQ==\\\"}\",\"lang\":\"opensearch_compounded_script\"}}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] - } -} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/3595.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/3595.yml index 581b1b2ca39..84fd0047d9c 100644 --- a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/3595.yml +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/3595.yml @@ -43,7 +43,7 @@ teardown: Content-Type: 'application/json' ppl: body: - query: 'source=logs | lookup region_info regionId' + query: 'source=logs | lookup region_info regionId | sort regionId' - match: {"total": 1} - match: {"schema": [{"name": "action", "type": "string"}, {"name": "regionId", "type": "string"}, {"name": "timestamp", "type": "timestamp"}, {"name": "regionName", "type": "string"}]} - match: {"datarows": [["login_attempt", "1", "2024-04-29 10:00:00", "us-east-1"]]} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java index 77d99507adf..f3248835d6b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/functions/GeoIpFunction.java @@ -90,6 +90,12 @@ public Expression implement( dataSource, ipAddress.toString(), Collections.emptySet(), nodeClient); } + public static Map fetchIpEnrichment( + String dataSource, Object ipAddress, NodeClient nodeClient) { + return fetchIpEnrichment( + dataSource, ipAddress.toString(), Collections.emptySet(), nodeClient); + } + public static Map fetchIpEnrichment( String dataSource, ExprIpValue ipAddress, diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index b63002d5456..f44b860701f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -51,7 +51,6 @@ import java.util.Collections; import java.util.GregorianCalendar; import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -59,7 +58,6 @@ import java.util.function.Supplier; import java.util.stream.Collectors; import lombok.Getter; -import org.apache.calcite.DataContext.Variable; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.type.RelDataType; @@ -69,7 +67,6 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexUnknownAs; import org.apache.calcite.rex.RexVisitorImpl; -import org.apache.calcite.runtime.Hook; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.SqlSyntax; @@ -108,6 +105,7 @@ import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.QueryStringQuery; import org.opensearch.sql.opensearch.storage.script.filter.lucene.relevance.SimpleQueryStringQuery; import org.opensearch.sql.opensearch.storage.serde.RelJsonSerializer; +import org.opensearch.sql.opensearch.storage.serde.ScriptParameterHelper; import org.opensearch.sql.opensearch.storage.serde.SerializationWrapper; /** @@ -1464,9 +1462,9 @@ private static String ipValueForPushDown(String value) { public static class ScriptQueryExpression extends QueryExpression { private RexNode analyzedNode; - // use lambda to generate code lazily to avoid store generated code private final Supplier codeGenerator; - private final Map params; + private String generatedCode; + private final ScriptParameterHelper parameterHelper; public ScriptQueryExpression( RexNode rexNode, @@ -1483,11 +1481,21 @@ public ScriptQueryExpression( } accumulateScriptCount(1); RelJsonSerializer serializer = new RelJsonSerializer(cluster); + this.parameterHelper = new ScriptParameterHelper(rowType.getFieldList(), fieldTypes, params); this.codeGenerator = () -> SerializationWrapper.wrapWithLangType( - ScriptEngineType.CALCITE, serializer.serialize(rexNode, rowType, fieldTypes)); - this.params = params; + ScriptEngineType.CALCITE, serializer.serialize(rexNode, parameterHelper)); + } + + // For filter script, this method will be called after planning phase; + // For the agg-script, this will be called in planning phase to generate agg builder. + // TODO: make agg-script lazy as well + private String getOrCreateGeneratedCode() { + if (generatedCode == null) { + generatedCode = codeGenerator.get(); + } + return generatedCode; } @Override @@ -1496,19 +1504,12 @@ public QueryBuilder builder() { } public Script getScript() { - long currentTime = Hook.CURRENT_TIME.get(-1L); - if (currentTime < 0) { - throw new UnsupportedScriptException( - "ScriptQueryExpression requires a valid current time from hook, but it is not set"); - } - Map mergedParams = new LinkedHashMap<>(params); - mergedParams.put(Variable.UTC_TIMESTAMP.camelName, currentTime); return new Script( DEFAULT_SCRIPT_TYPE, COMPOUNDED_LANG_NAME, - codeGenerator.get(), + getOrCreateGeneratedCode(), Collections.emptyMap(), - mergedParams); + this.parameterHelper.getParameters()); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java index 94700dfa6ee..353ad12b608 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java @@ -31,6 +31,7 @@ import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.TableScan; +import org.apache.calcite.rel.externalize.RelWriterImpl; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.metadata.RelMdUtil; @@ -96,7 +97,11 @@ public RelDataType deriveRowType() { @Override public RelWriter explainTerms(RelWriter pw) { - String explainString = pushDownContext + ", " + pushDownContext.getRequestBuilder(); + String explainString = String.valueOf(pushDownContext); + if (pw instanceof RelWriterImpl) { + // Only add request builder to the explain plan + explainString += ", " + pushDownContext.createRequestBuilder(); + } return super.explainTerms(pw) .itemIf("PushDownContext", explainString, !pushDownContext.isEmpty()); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java index ca2370dfa98..882f8b53c7f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java @@ -115,7 +115,7 @@ public Result implement(EnumerableRelImplementor implementor, Prefer pref) { return new AbstractEnumerable<>() { @Override public Enumerator enumerator() { - OpenSearchRequestBuilder requestBuilder = getOrCreateRequestBuilder(); + OpenSearchRequestBuilder requestBuilder = pushDownContext.createRequestBuilder(); return new OpenSearchIndexEnumerator( osIndex.getClient(), getFieldPath(), @@ -132,28 +132,4 @@ private List getFieldPath() { .map(f -> osIndex.getAliasMapping().getOrDefault(f, f)) .collect(Collectors.toList()); } - - /** - * In some edge cases where the digests of more than one scan are the same, and then the Calcite - * planner will reuse the same scan along with the same PushDownContext inner it. However, the - * `OpenSearchRequestBuilder` inner `PushDownContext` is not reusable since it has status changed - * in the search process. - * - *

To avoid this issue and try to construct `OpenSearchRequestBuilder` as less as possible, - * this method will get and reuse the `OpenSearchRequestBuilder` in PushDownContext for the first - * time, and then construct new ones for the following invoking. - * - * @return OpenSearchRequestBuilder to be used by enumerator - */ - private volatile boolean isRequestBuilderUsedByEnumerator = false; - - private OpenSearchRequestBuilder getOrCreateRequestBuilder() { - synchronized (this.pushDownContext) { - if (isRequestBuilderUsedByEnumerator) { - return this.pushDownContext.createRequestBuilder(); - } - isRequestBuilderUsedByEnumerator = true; - return this.pushDownContext.getRequestBuilder(); - } - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java index 4be4d5033a6..a0ddbf89348 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java @@ -5,6 +5,8 @@ package org.opensearch.sql.opensearch.storage.scan; +import static org.opensearch.sql.opensearch.storage.serde.ScriptParameterHelper.MISSING_MAX; + import com.google.common.collect.ImmutableList; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -50,7 +52,6 @@ import org.opensearch.search.sort.SortOrder; import org.opensearch.sql.ast.expression.Argument; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; -import org.opensearch.sql.calcite.utils.PlanUtils; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; @@ -486,9 +487,12 @@ public CalciteLogicalIndexScan pushdownSortExpr(List sortExprDig } RexNode sortExpr = digest.getExpression(); assert sortExpr instanceof RexCall : "sort expression should be RexCall"; - Map directionParams = new LinkedHashMap<>(); - directionParams.put(PlanUtils.NULL_DIRECTION, digest.getNullDirection().name()); - directionParams.put(PlanUtils.DIRECTION, digest.getDirection().name()); + Map missingValueParams = + new LinkedHashMap<>() { + { + put(MISSING_MAX, digest.isMissingMax()); + } + }; // Complex expression - use ScriptQueryExpression to generate script for sort PredicateAnalyzer.ScriptQueryExpression scriptExpr = new PredicateAnalyzer.ScriptQueryExpression( @@ -496,7 +500,7 @@ public CalciteLogicalIndexScan pushdownSortExpr(List sortExprDig rowType, osIndex.getAllFieldTypes(), getCluster(), - directionParams); + missingValueParams); // Determine the correct ScriptSortType based on the expression's return type ScriptSortType sortType = getScriptSortType(sortExpr.getType()); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AbstractAction.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AbstractAction.java index 65ef6233ffb..9ddba2f970b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AbstractAction.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AbstractAction.java @@ -14,10 +14,10 @@ public interface AbstractAction { void apply(T target); /** - * Apply the action on the target T and add the operation to the context + * Add the operation to the context * * @param context the context to add the operation to * @param operation the operation to add to the context */ - void transform(PushDownContext context, PushDownOperation operation); + void pushOperation(PushDownContext context, PushDownOperation operation); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggregationBuilderAction.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggregationBuilderAction.java index cd3e84bf7cf..f2362e3c1cc 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggregationBuilderAction.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggregationBuilderAction.java @@ -7,13 +7,9 @@ /** A lambda action to apply on the {@link AggPushDownAction} */ public interface AggregationBuilderAction extends AbstractAction { - /** - * Apply the action on the target {@link AggPushDownAction} and add the operation to the context - * - * @param context the context to add the operation to - * @param operation the operation to add to the context - */ - default void transform(PushDownContext context, PushDownOperation operation) { + default void pushOperation(PushDownContext context, PushDownOperation operation) { + // Apply transformation to aggregation builder in the optimization phase as some transformation + // may cause exception. We need to detect that exception in advance. apply(context.getAggPushDownAction()); context.getOperationsForAgg().add(operation); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/OSRequestBuilderAction.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/OSRequestBuilderAction.java index bba33883b49..6abe97266f7 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/OSRequestBuilderAction.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/OSRequestBuilderAction.java @@ -9,15 +9,7 @@ /** A lambda action to apply on the {@link OpenSearchRequestBuilder} */ public interface OSRequestBuilderAction extends AbstractAction { - /** - * Apply the action on the target {@link OpenSearchRequestBuilder} and add the operation to the - * context - * - * @param context the context to add the operation to - * @param operation the operation to add to the context - */ - default void transform(PushDownContext context, PushDownOperation operation) { - apply(context.getRequestBuilder()); + default void pushOperation(PushDownContext context, PushDownOperation operation) { context.getOperationsForRequestBuilder().add(operation); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/PushDownContext.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/PushDownContext.java index 4a2ade440cf..29700fd6606 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/PushDownContext.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/PushDownContext.java @@ -13,19 +13,22 @@ import lombok.Getter; import org.jetbrains.annotations.NotNull; import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder; +import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder.PushDownUnSupportedException; import org.opensearch.sql.opensearch.storage.OpenSearchIndex; /** Push down context is used to store all the push down operations that are applied to the query */ @Getter public class PushDownContext extends AbstractCollection { private final OpenSearchIndex osIndex; - private final OpenSearchRequestBuilder requestBuilder; private ArrayDeque operationsForRequestBuilder; private boolean isAggregatePushed = false; private AggPushDownAction aggPushDownAction; private ArrayDeque operationsForAgg; + // Records the start pos of the query, which is updated by new added limit operations. + private int startFrom = 0; + private boolean isLimitPushed = false; private boolean isProjectPushed = false; private boolean isMeasureOrderPushed = false; @@ -36,7 +39,6 @@ public class PushDownContext extends AbstractCollection { public PushDownContext(OpenSearchIndex osIndex) { this.osIndex = osIndex; - this.requestBuilder = osIndex.createRequestBuilder(); } @Override @@ -95,12 +97,19 @@ ArrayDeque getOperationsForAgg() { @Override public boolean add(PushDownOperation operation) { - operation.action().transform(this, operation); + operation.action().pushOperation(this, operation); if (operation.type() == PushDownType.AGGREGATION) { isAggregatePushed = true; this.aggPushDownAction = (AggPushDownAction) operation.action(); } if (operation.type() == PushDownType.LIMIT) { + startFrom += ((LimitDigest) operation.digest()).offset(); + if (startFrom >= osIndex.getMaxResultWindow()) { + throw new PushDownUnSupportedException( + String.format( + "[INTERNAL] Requested offset %d should be less than the max result window %d", + startFrom, osIndex.getMaxResultWindow())); + } isLimitPushed = true; if (isSortPushed || isMeasureOrderPushed || isSortExprPushed) { isTopKPushed = true; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/SortExprDigest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/SortExprDigest.java index 5b7fb1db320..cde43634ae2 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/SortExprDigest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/SortExprDigest.java @@ -9,6 +9,8 @@ import lombok.AllArgsConstructor; import lombok.Getter; import org.apache.calcite.rel.RelFieldCollation; +import org.apache.calcite.rel.RelFieldCollation.Direction; +import org.apache.calcite.rel.RelFieldCollation.NullDirection; import org.apache.calcite.rex.RexNode; import org.apache.commons.lang3.StringUtils; import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; @@ -102,4 +104,8 @@ public String toString() { return String.format( "%s %s NULLS_%s", sortTarget, direction.toString(), nullDirection.toString()); } + + public boolean isMissingMax() { + return (direction == Direction.ASCENDING) ^ (nullDirection == NullDirection.FIRST); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/CalciteScriptEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/CalciteScriptEngine.java index aac14a16dc2..5f4481a1209 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/CalciteScriptEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/CalciteScriptEngine.java @@ -27,42 +27,37 @@ package org.opensearch.sql.opensearch.storage.script; -import static org.opensearch.sql.data.type.ExprCoreType.BYTE; -import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; -import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; -import static org.opensearch.sql.data.type.ExprCoreType.IP; -import static org.opensearch.sql.data.type.ExprCoreType.SHORT; +import static org.opensearch.sql.opensearch.storage.serde.ScriptParameterHelper.DIGESTS; +import static org.opensearch.sql.opensearch.storage.serde.ScriptParameterHelper.SOURCES; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import java.lang.reflect.Type; import java.time.chrono.ChronoZonedDateTime; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.BiFunction; +import java.util.stream.Collectors; +import lombok.Getter; import lombok.RequiredArgsConstructor; import org.apache.calcite.DataContext; -import org.apache.calcite.adapter.enumerable.EnumUtils; import org.apache.calcite.adapter.enumerable.PhysType; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; -import org.apache.calcite.adapter.enumerable.RexToLixTranslator.InputGetter; import org.apache.calcite.adapter.java.JavaTypeFactory; import org.apache.calcite.config.CalciteSystemProperty; import org.apache.calcite.jdbc.JavaTypeFactoryImpl; import org.apache.calcite.linq4j.QueryProvider; import org.apache.calcite.linq4j.function.Function1; import org.apache.calcite.linq4j.tree.BlockBuilder; -import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.linq4j.tree.LabelTarget; -import org.apache.calcite.linq4j.tree.MethodCallExpression; import org.apache.calcite.linq4j.tree.MethodDeclaration; import org.apache.calcite.linq4j.tree.ParameterExpression; -import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelRecordType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexExecutable; import org.apache.calcite.rex.RexNode; @@ -82,12 +77,7 @@ import org.opensearch.script.ScriptEngine; import org.opensearch.script.StringSortScript; import org.opensearch.search.lookup.SourceLookup; -import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; -import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.model.ExprTimestampValue; -import org.opensearch.sql.data.type.ExprCoreType; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; import org.opensearch.sql.opensearch.storage.script.aggregation.CalciteAggregationScriptFactory; import org.opensearch.sql.opensearch.storage.script.filter.CalciteFilterScriptFactory; import org.opensearch.sql.opensearch.storage.script.sort.CalciteNumberSortScriptFactory; @@ -131,17 +121,19 @@ public String getType() { @Override public T compile( String scriptName, String scriptCode, ScriptContext context, Map options) { - Map objectMap = relJsonSerializer.deserialize(scriptCode); - RexNode rexNode = (RexNode) objectMap.get(RelJsonSerializer.EXPR); - RelDataType rowType = (RelDataType) objectMap.get(RelJsonSerializer.ROW_TYPE); - Map fieldTypes = - (Map) objectMap.get(RelJsonSerializer.FIELD_TYPES); - - JavaTypeFactory typeFactory = OpenSearchTypeFactory.TYPE_FACTORY; - RexToLixTranslator.InputGetter getter = new ScriptInputGetter(typeFactory, rowType, fieldTypes); + RexNode rexNode = relJsonSerializer.deserialize(scriptCode); + + RexToLixTranslator.InputGetter getter = + (blockBuilder, i, type) -> { + throw new UnsupportedScriptException( + "[BUG]There shouldn't be RexInputRef in the RexNode."); + }; String code = CalciteScriptEngine.translate( - relJsonSerializer.getCluster().getRexBuilder(), List.of(rexNode), getter, rowType); + relJsonSerializer.getCluster().getRexBuilder(), + List.of(rexNode), + getter, + new RelRecordType(List.of())); Function1 function = new RexExecutable(code, "generated Rex code").getFunction(); @@ -172,89 +164,27 @@ public UnsupportedScriptException(Throwable cause) { } } - /** - * Implementation of {@link org.apache.calcite.adapter.enumerable.RexToLixTranslator.InputGetter} - * that reads the values of input fields by calling - * {@link org.apache.calcite.DataContext#get}("inputRecord"). - */ - public static class ScriptInputGetter implements InputGetter { - private final RelDataTypeFactory typeFactory; - private final RelDataType rowType; - private final Map fieldTypes; - - public ScriptInputGetter( - RelDataTypeFactory typeFactory, RelDataType rowType, Map fieldTypes) { - this.typeFactory = typeFactory; - this.rowType = rowType; - this.fieldTypes = fieldTypes; - } - - @Override - public org.apache.calcite.linq4j.tree.Expression field( - BlockBuilder list, int index, @Nullable Type storageType) { - String fieldName = rowType.getFieldList().get(index).getName(); - ExprType exprType = fieldTypes.get(fieldName); - String referenceField = OpenSearchTextType.toKeywordSubField(fieldName, exprType); - MethodCallExpression fieldValueExpr = - // Have to invoke `getFromSource` if the field is the text without keyword or struct - (referenceField == null || exprType == ExprCoreType.STRUCT) - ? Expressions.call( - EnumUtils.convert(DataContext.ROOT, ScriptDataContext.class), - Types.lookupMethod(ScriptDataContext.class, "getFromSource", String.class), - Expressions.constant(fieldName)) - : Expressions.call( - DataContext.ROOT, - BuiltInMethod.DATA_CONTEXT_GET.method, - Expressions.constant(referenceField)); - if (storageType == null) { - final RelDataType fieldType = rowType.getFieldList().get(index).getType(); - storageType = ((JavaTypeFactory) typeFactory).getJavaClass(fieldType); - } - return EnumUtils.convert(tryConvertDocValue(fieldValueExpr, exprType), storageType); - } - - /** - * DocValue only support long and double for integer and float, cast to the related type first - */ - private Expression tryConvertDocValue(Expression docValueExpr, ExprType exprType) { - Expression docValue = docValueExpr; - if (exprType instanceof ExprCoreType) { - ExprCoreType type = (ExprCoreType) exprType; - switch (type) { - case INTEGER: - case SHORT: - case BYTE: - docValue = EnumUtils.convert(docValueExpr, Long.class); - break; - case FLOAT: - docValue = EnumUtils.convert(docValueExpr, Double.class); - break; - // IP is scanned in as a string but used as ExprIpValue later. We call the constructor - // beforehand. - case IP: - docValue = Expressions.new_( - ExprIpValue.class, EnumUtils.convert(docValueExpr, String.class)); - default: - // fallthrough - } - } - return docValue; - } - } - public static class ScriptDataContext implements DataContext { private final Map> docProvider; private final SourceLookup sourceLookup; - private final Map params; + private final long utcTimestamp; + private final List sources; + private final List digests; + private final Map parameterToIndex; public ScriptDataContext( Map> docProvider, SourceLookup sourceLookup, - Map params) { + Map params, + Map parameterToIndex) { this.docProvider = docProvider; this.sourceLookup = sourceLookup; - this.params = params; + this.utcTimestamp = (long) params.get(Variable.UTC_TIMESTAMP.camelName); + this.sources = ((List) params.get(SOURCES)).stream().map(Source::fromValue).collect( + Collectors.toList()); + this.digests = (List) params.get(DIGESTS); + this.parameterToIndex = parameterToIndex; } @Override @@ -275,9 +205,26 @@ public QueryProvider getQueryProvider() { @Override public Object get(String name) { // UTC_TIMESTAMP is a special variable used for some time related functions. - if (Variable.UTC_TIMESTAMP.camelName.equals(name)) - return params.get(Variable.UTC_TIMESTAMP.camelName); + if (Variable.UTC_TIMESTAMP.camelName.equals(name)) return this.utcTimestamp; + + try { + int index = parameterToIndex.get(name); + switch (sources.get(index)) { + case DOC_VALUE: + return getFromDocValue((String) digests.get(index)); + case SOURCE: + return getFromSource((String) digests.get(index)); + case LITERAL: + return digests.get(index); + default: + throw new IllegalStateException("Unknown source type for parameter at index: " + index); + } + } catch (Exception e) { + throw new IllegalStateException("Failed to get value for parameter " + name); + } + } + public Object getFromDocValue(String name) { ScriptDocValues docValue = this.docProvider.get(name); if (docValue == null || docValue.isEmpty()) { return null; // No way to differentiate null and missing from doc value @@ -298,6 +245,35 @@ public Object getFromSource(String name) { } } + @Getter + public enum Source { + DOC_VALUE(0), + SOURCE(1), + LITERAL(2); + + private final int value; + + Source(int value) { + this.value = value; + } + + private static final Map VALUE_TO_SOURCE = new HashMap<>(); + + static { + for (Source source : Source.values()) { + VALUE_TO_SOURCE.put(source.value, source); + } + } + + public static Source fromValue(int value) { + Source source = VALUE_TO_SOURCE.get(value); + if (source == null) { + throw new IllegalArgumentException("No Source with value: " + value); + } + return source; + } + } + /** * This function is copied from Calcite RexExecutorImpl It's used to compile RexNode expression to * java code string. diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/CalciteAggregationScript.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/CalciteAggregationScript.java index 87133a37758..7c09a48a06c 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/CalciteAggregationScript.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/CalciteAggregationScript.java @@ -39,17 +39,21 @@ class CalciteAggregationScript extends AggregationScript { private final RelDataType type; + private final Map parametersToIndex; + public CalciteAggregationScript( Function1 function, RelDataType type, SearchLookup lookup, LeafReaderContext context, - Map params) { + Map params, + Map parametersToIndex) { super(params, lookup, context); this.context = context; this.calciteScript = new CalciteScript(function, params); this.sourceLookup = lookup.getLeafSearchLookup(context).source(); this.type = type; + this.parametersToIndex = parametersToIndex; } @Override @@ -60,7 +64,8 @@ public void setDocument(int docid) { @Override public Object execute() { - Object value = calciteScript.execute(this.getDoc(), this.sourceLookup)[0]; + Object value = + calciteScript.execute(this.getDoc(), this.sourceLookup, this.parametersToIndex)[0]; ExprType exprType = OpenSearchTypeFactory.convertRelDataTypeToExprType(type); // See logic in {@link ExpressionAggregationScript::execute} ExprCoreType coreType = (ExprCoreType) exprType; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/CalciteAggregationScriptLeafFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/CalciteAggregationScriptLeafFactory.java index 8555b0a82a7..b0b0914e072 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/CalciteAggregationScriptLeafFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/CalciteAggregationScriptLeafFactory.java @@ -5,7 +5,12 @@ package org.opensearch.sql.opensearch.storage.script.aggregation; +import static org.opensearch.sql.opensearch.storage.serde.ScriptParameterHelper.SOURCES; + +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.apache.calcite.DataContext; import org.apache.calcite.linq4j.function.Function1; import org.apache.calcite.rel.type.RelDataType; @@ -25,6 +30,12 @@ class CalciteAggregationScriptLeafFactory implements AggregationScript.LeafFacto /** Document lookup that returns doc values. */ private final SearchLookup lookup; + /** + * Stores the parameter names to the actual indices in SOURCES. Generate it in advance in factory + * to save the process for each document* + */ + private final Map parametersToIndex; + public CalciteAggregationScriptLeafFactory( Function1 function, RelDataType type, @@ -34,11 +45,15 @@ public CalciteAggregationScriptLeafFactory( this.type = type; this.params = params; this.lookup = lookup; + this.parametersToIndex = + IntStream.range(0, ((List) params.get(SOURCES)).size()) + .boxed() + .collect(Collectors.toMap(i -> "?" + i, i -> i)); } @Override public AggregationScript newInstance(LeafReaderContext ctx) { - return new CalciteAggregationScript(function, type, lookup, ctx, params); + return new CalciteAggregationScript(function, type, lookup, ctx, params, parametersToIndex); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/CalciteScript.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/CalciteScript.java index 4568fddc8ed..ae79f18fa45 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/CalciteScript.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/CalciteScript.java @@ -40,9 +40,14 @@ public CalciteScript(Function1 function, Map> docProvider, SourceLookup sourceLookup) { + public Object[] execute( + Map> docProvider, + SourceLookup sourceLookup, + Map parametersToIndex) { return AccessController.doPrivileged( (PrivilegedAction) - () -> function.apply(new ScriptDataContext(docProvider, sourceLookup, params))); + () -> + function.apply( + new ScriptDataContext(docProvider, sourceLookup, params, parametersToIndex))); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/CalciteFilterScript.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/CalciteFilterScript.java index 831b4874d9e..122346ea6e9 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/CalciteFilterScript.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/CalciteFilterScript.java @@ -29,16 +29,20 @@ class CalciteFilterScript extends FilterScript { private final SourceLookup sourceLookup; + private final Map parametersToIndex; + public CalciteFilterScript( Function1 function, SearchLookup lookup, LeafReaderContext context, - Map params) { + Map params, + Map parametersToIndex) { super(params, lookup, context); this.context = context; this.calciteScript = new CalciteScript(function, params); // TODO: we'd better get source from the leafLookup of super once it's available this.sourceLookup = lookup.getLeafSearchLookup(context).source(); + this.parametersToIndex = parametersToIndex; } @Override @@ -49,7 +53,8 @@ public void setDocument(int docid) { @Override public boolean execute() { - Object result = calciteScript.execute(this.getDoc(), this.sourceLookup)[0]; + Object result = + calciteScript.execute(this.getDoc(), this.sourceLookup, this.parametersToIndex)[0]; // The result should be type of BOOLEAN_NULLABLE. Treat it as false if null return result != null && (boolean) result; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/CalciteFilterScriptLeafFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/CalciteFilterScriptLeafFactory.java index 93f538a50b6..0158e593902 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/CalciteFilterScriptLeafFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/filter/CalciteFilterScriptLeafFactory.java @@ -5,7 +5,12 @@ package org.opensearch.sql.opensearch.storage.script.filter; +import static org.opensearch.sql.opensearch.storage.serde.ScriptParameterHelper.SOURCES; + +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.apache.calcite.DataContext; import org.apache.calcite.linq4j.function.Function1; import org.apache.lucene.index.LeafReaderContext; @@ -23,15 +28,25 @@ class CalciteFilterScriptLeafFactory implements FilterScript.LeafFactory { /** Document lookup that returns doc values. */ private final SearchLookup lookup; + /** + * Stores the parameter names to the actual indices in SOURCES. Generate it in advance in factory + * to save the process for each document* + */ + private final Map parametersToIndex; + public CalciteFilterScriptLeafFactory( Function1 function, Map params, SearchLookup lookup) { this.function = function; this.params = params; this.lookup = lookup; + this.parametersToIndex = + IntStream.range(0, ((List) params.get(SOURCES)).size()) + .boxed() + .collect(Collectors.toMap(i -> "?" + i, i -> i)); } @Override public FilterScript newInstance(LeafReaderContext ctx) { - return new CalciteFilterScript(function, lookup, ctx, params); + return new CalciteFilterScript(function, lookup, ctx, params, parametersToIndex); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/CalciteNumberSortScript.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/CalciteNumberSortScript.java index c1c48cd64bc..345502ec1cf 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/CalciteNumberSortScript.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/CalciteNumberSortScript.java @@ -5,17 +5,16 @@ package org.opensearch.sql.opensearch.storage.script.sort; +import static org.opensearch.sql.opensearch.storage.serde.ScriptParameterHelper.MISSING_MAX; + import java.util.Map; import lombok.EqualsAndHashCode; import org.apache.calcite.DataContext; import org.apache.calcite.linq4j.function.Function1; -import org.apache.calcite.rel.RelFieldCollation.Direction; -import org.apache.calcite.rel.RelFieldCollation.NullDirection; import org.apache.lucene.index.LeafReaderContext; import org.opensearch.script.NumberSortScript; import org.opensearch.search.lookup.SearchLookup; import org.opensearch.search.lookup.SourceLookup; -import org.opensearch.sql.calcite.utils.PlanUtils; import org.opensearch.sql.opensearch.storage.script.core.CalciteScript; /** Calcite number sort script. */ @@ -27,27 +26,22 @@ public class CalciteNumberSortScript extends NumberSortScript { private final SourceLookup sourceLookup; private final LeafReaderContext context; - private final Direction direction; - private final NullDirection nullDirection; + private final boolean missingMax; + private final Map parametersToIndex; public CalciteNumberSortScript( Function1 function, SearchLookup lookup, LeafReaderContext context, - Map params) { + Map params, + Map parametersToIndex) { super(params, lookup, context); this.calciteScript = new CalciteScript(function, params); // TODO: we'd better get source from the leafLookup of super once it's available this.sourceLookup = lookup.getLeafSearchLookup(context).source(); this.context = context; - this.direction = - params.containsKey(PlanUtils.DIRECTION) - ? Direction.valueOf((String) params.get(PlanUtils.DIRECTION)) - : Direction.ASCENDING; - this.nullDirection = - params.containsKey(PlanUtils.NULL_DIRECTION) - ? NullDirection.valueOf((String) params.get(PlanUtils.NULL_DIRECTION)) - : NullDirection.FIRST; + this.parametersToIndex = parametersToIndex; + this.missingMax = (Boolean) params.getOrDefault(MISSING_MAX, false); } @Override @@ -58,14 +52,13 @@ public void setDocument(int docid) { @Override public double execute() { - Object value = calciteScript.execute(this.getDoc(), this.sourceLookup)[0]; + Object value = + calciteScript.execute(this.getDoc(), this.sourceLookup, this.parametersToIndex)[0]; // There is a limitation here when the Double value is exactly theoretical min/max value. // It can't distinguish the ordering between null and exact Double.NEGATIVE_INFINITY or // Double.NaN. if (value == null) { - boolean isAscending = direction == Direction.ASCENDING; - boolean isNullFirst = nullDirection == NullDirection.FIRST; - return isAscending == isNullFirst ? Double.NEGATIVE_INFINITY : Double.NaN; + return this.missingMax ? Double.NaN : Double.NEGATIVE_INFINITY; } return ((Number) value).doubleValue(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/CalciteNumberSortScriptLeafFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/CalciteNumberSortScriptLeafFactory.java index 703a8946e43..f3c94162cdc 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/CalciteNumberSortScriptLeafFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/CalciteNumberSortScriptLeafFactory.java @@ -5,8 +5,13 @@ package org.opensearch.sql.opensearch.storage.script.sort; +import static org.opensearch.sql.opensearch.storage.serde.ScriptParameterHelper.SOURCES; + import java.io.IOException; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import lombok.EqualsAndHashCode; import org.apache.calcite.DataContext; import org.apache.calcite.linq4j.function.Function1; @@ -26,16 +31,26 @@ class CalciteNumberSortScriptLeafFactory implements NumberSortScript.LeafFactory /** Document lookup that returns doc values. */ private final SearchLookup lookup; + /** + * Stores the parameter names to the actual indices in SOURCES. Generate it in advance in factory + * to save the process for each document* + */ + private final Map parametersToIndex; + public CalciteNumberSortScriptLeafFactory( Function1 function, Map params, SearchLookup lookup) { this.function = function; this.params = params; this.lookup = lookup; + this.parametersToIndex = + IntStream.range(0, ((List) params.get(SOURCES)).size()) + .boxed() + .collect(Collectors.toMap(i -> "?" + i, i -> i)); } @Override public NumberSortScript newInstance(LeafReaderContext context) throws IOException { - return new CalciteNumberSortScript(function, lookup, context, params); + return new CalciteNumberSortScript(function, lookup, context, params, parametersToIndex); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/CalciteStringSortScript.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/CalciteStringSortScript.java index 809306247b2..e21202431ae 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/CalciteStringSortScript.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/CalciteStringSortScript.java @@ -5,17 +5,16 @@ package org.opensearch.sql.opensearch.storage.script.sort; +import static org.opensearch.sql.opensearch.storage.serde.ScriptParameterHelper.MISSING_MAX; + import java.util.Map; import lombok.EqualsAndHashCode; import org.apache.calcite.DataContext; import org.apache.calcite.linq4j.function.Function1; -import org.apache.calcite.rel.RelFieldCollation.Direction; -import org.apache.calcite.rel.RelFieldCollation.NullDirection; import org.apache.lucene.index.LeafReaderContext; import org.opensearch.script.StringSortScript; import org.opensearch.search.lookup.SearchLookup; import org.opensearch.search.lookup.SourceLookup; -import org.opensearch.sql.calcite.utils.PlanUtils; import org.opensearch.sql.opensearch.storage.script.core.CalciteScript; /** Calcite string sort script. */ @@ -27,8 +26,9 @@ public class CalciteStringSortScript extends StringSortScript { private final SourceLookup sourceLookup; private final LeafReaderContext context; - private final Direction direction; - private final NullDirection nullDirection; + private final Map parametersToIndex; + + private final boolean missingMax; private static final String MAX_SENTINEL = "\uFFFF\uFFFF_NULL_PLACEHOLDER_"; private static final String MIN_SENTINEL = "\u0000\u0000_NULL_PLACEHOLDER_"; @@ -37,20 +37,15 @@ public CalciteStringSortScript( Function1 function, SearchLookup lookup, LeafReaderContext context, - Map params) { + Map params, + Map parametersToIndex) { super(params, lookup, context); this.calciteScript = new CalciteScript(function, params); // TODO: we'd better get source from the leafLookup of super once it's available this.sourceLookup = lookup.getLeafSearchLookup(context).source(); this.context = context; - this.direction = - params.containsKey(PlanUtils.DIRECTION) - ? Direction.valueOf((String) params.get(PlanUtils.DIRECTION)) - : Direction.ASCENDING; - this.nullDirection = - params.containsKey(PlanUtils.NULL_DIRECTION) - ? NullDirection.valueOf((String) params.get(PlanUtils.NULL_DIRECTION)) - : NullDirection.FIRST; + this.parametersToIndex = parametersToIndex; + this.missingMax = (Boolean) params.getOrDefault(MISSING_MAX, false); } @Override @@ -61,13 +56,11 @@ public void setDocument(int docid) { @Override public String execute() { - Object value = calciteScript.execute(this.getDoc(), this.sourceLookup)[0]; + Object value = calciteScript.execute(this.getDoc(), this.sourceLookup, parametersToIndex)[0]; // There is a limitation here when the String value is larger or smaller than sentinel values. // It can't guarantee the lexigraphic ordering between null and special strings. if (value == null) { - boolean isAscending = direction == Direction.ASCENDING; - boolean isNullFirst = nullDirection == NullDirection.FIRST; - return isAscending == isNullFirst ? MIN_SENTINEL : MAX_SENTINEL; + return this.missingMax ? MAX_SENTINEL : MIN_SENTINEL; } return value.toString(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/CalciteStringSortScriptLeafFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/CalciteStringSortScriptLeafFactory.java index a95ee30d59d..36a88fa6b0c 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/CalciteStringSortScriptLeafFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/sort/CalciteStringSortScriptLeafFactory.java @@ -5,8 +5,13 @@ package org.opensearch.sql.opensearch.storage.script.sort; +import static org.opensearch.sql.opensearch.storage.serde.ScriptParameterHelper.SOURCES; + import java.io.IOException; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import lombok.EqualsAndHashCode; import org.apache.calcite.DataContext; import org.apache.calcite.linq4j.function.Function1; @@ -27,15 +32,25 @@ public class CalciteStringSortScriptLeafFactory implements StringSortScript.Leaf /** Search lookup. */ private final SearchLookup lookup; + /** + * Stores the parameter names to the actual indices in SOURCES. Generate it in advance in factory + * to save the process for each document* + */ + private final Map parametersToIndex; + public CalciteStringSortScriptLeafFactory( Function1 function, Map params, SearchLookup lookup) { this.function = function; this.params = params; this.lookup = lookup; + this.parametersToIndex = + IntStream.range(0, ((List) params.get(SOURCES)).size()) + .boxed() + .collect(Collectors.toMap(i -> "?" + i, i -> i)); } @Override public StringSortScript newInstance(LeafReaderContext context) throws IOException { - return new CalciteStringSortScript(function, lookup, context, params); + return new CalciteStringSortScript(function, lookup, context, params, parametersToIndex); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java index c7451adbf8b..d77dee3e297 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java @@ -79,6 +79,7 @@ import org.checkerframework.checker.nullness.qual.PolyNull; import org.opensearch.sql.calcite.type.AbstractExprRelDataType; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT; /** * An extension to {@link RelJson} to allow serialization & deserialization of UDTs @@ -364,7 +365,11 @@ public RelDataType toType(RelDataTypeFactory typeFactory, Object o) { // Reconstruct UDT from its udt tag Object udtName = ((Map) o).get("udt"); OpenSearchTypeFactory.ExprUDT udt = OpenSearchTypeFactory.ExprUDT.valueOf((String) udtName); - return ((OpenSearchTypeFactory) typeFactory).createUDT(udt); + // View IP as string to avoid using a value of customized java type in the script. + if (udt == ExprUDT.EXPR_IP) return super.toType(typeFactory, o); + RelDataType type = ((OpenSearchTypeFactory) typeFactory).createUDT(udt); + boolean nullable = (Boolean) ((Map) o).get("nullable"); + return typeFactory.createTypeWithNullability(type, nullable); } return super.toType(typeFactory, o); } @@ -802,4 +807,10 @@ public boolean getBoolean(String tag, boolean default_) { throw new UnsupportedOperationException(); } } + + static RexNode translateInput( + RelJson relJson, int input, Map map, RelInput relInput) { + throw new UnsupportedOperationException( + "There shouldn't be any RexInputRef in the serialized RexNode."); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/OpenSearchRelInputTranslator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/OpenSearchRelInputTranslator.java index b5f401bb64a..da69c2ad518 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/OpenSearchRelInputTranslator.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/OpenSearchRelInputTranslator.java @@ -36,7 +36,7 @@ public RexNode translateInput( if (input < rowType.getFieldCount()) { final RelDataTypeField field = rowType.getFieldList().get(input); - return rexBuilder.makeInputRef(field.getType(), input); + return rexBuilder.makeLocalRef(field.getType(), input); } throw new RuntimeException("input field " + input + " is out of range"); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializer.java index fb751c8a72c..604012c4870 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializer.java @@ -14,13 +14,11 @@ import java.io.ObjectOutputStream; import java.io.Serializable; import java.util.Base64; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import lombok.Getter; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.rel.externalize.RelJson; -import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlOperatorTable; import org.apache.calcite.sql.fun.SqlLibrary; @@ -28,12 +26,9 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.util.SqlOperatorTables; import org.apache.calcite.util.JsonBuilder; -import org.apache.commons.lang3.tuple.Pair; import org.opensearch.sql.calcite.CalcitePlanContext; -import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.function.PPLBuiltinOperators; -import org.opensearch.sql.opensearch.executor.OpenSearchExecutionEngine; -import org.opensearch.sql.opensearch.util.OpenSearchRelOptUtil; +import org.opensearch.sql.opensearch.executor.OpenSearchExecutionEngine.OperatorTable; /** * A serializer that (de-)serializes Calcite RexNode, RelDataType and OpenSearch field mapping. @@ -47,10 +42,6 @@ public class RelJsonSerializer { private final RelOptCluster cluster; - - public static final String EXPR = "expr"; - public static final String FIELD_TYPES = "fieldTypes"; - public static final String ROW_TYPE = "rowType"; private static final ObjectMapper mapper = new ObjectMapper(); private static final TypeReference> TYPE_REF = new TypeReference<>() {}; @@ -72,7 +63,7 @@ private static SqlOperatorTable getPplSqlOperatorTable() { SqlOperatorTables.chain( PPLBuiltinOperators.instance(), SqlStdOperatorTable.instance(), - OpenSearchExecutionEngine.OperatorTable.instance(), + OperatorTable.instance(), // Add a list of necessary SqlLibrary if needed SqlLibraryOperatorTableFactory.INSTANCE.getOperatorTable( SqlLibrary.MYSQL, @@ -89,45 +80,31 @@ private static SqlOperatorTable getPplSqlOperatorTable() { * Serializes Calcite expressions and field types into a map object string. * *

This method: - *

  • Convert RexNode and RelDataType objects to JSON strings. - *
  • Combines these JSON strings with OpenSearch field mappings into a map + *
  • Standardize the original RexNode + *
  • Convert RexNode objects to JSON strings. *
  • Encodes the resulting map into a final object string * * @param rexNode pushed down RexNode - * @param rowType row type of RexNode input - * @param fieldTypes input field and ExprType mapping - * @return serialized string of map structure for inputs + * @return serialized string of RexNode expression. */ - public String serialize(RexNode rexNode, RelDataType rowType, Map fieldTypes) { - // Extract necessary fields and remap expression input indices for original RexNode - Pair remappedRexInfo = - OpenSearchRelOptUtil.getRemappedRexAndType(rexNode, rowType); - Map filteredFieldTypes = new HashMap<>(); - for (String fieldName : remappedRexInfo.getValue().getFieldNames()) { - filteredFieldTypes.put(fieldName, fieldTypes.get(fieldName)); - } + public String serialize(RexNode rexNode, ScriptParameterHelper parameterHelper) { + RexNode standardizedRexExpr = + RexStandardizer.standardizeRexNodeExpression(rexNode, parameterHelper); try { // Serialize RexNode and RelDataType by JSON JsonBuilder jsonBuilder = new JsonBuilder(); RelJson relJson = ExtendedRelJson.create(jsonBuilder); - String rexNodeJson = jsonBuilder.toJsonString(relJson.toJson(remappedRexInfo.getKey())); - Object rowTypeJsonObj = relJson.toJson(remappedRexInfo.getValue()); - String rowTypeJson = jsonBuilder.toJsonString(rowTypeJsonObj); - // Construct envelope of serializable objects - Map envelope = - Map.of(EXPR, rexNodeJson, FIELD_TYPES, filteredFieldTypes, ROW_TYPE, rowTypeJson); + String rexNodeJson = jsonBuilder.toJsonString(relJson.toJson(standardizedRexExpr)); + if (CalcitePlanContext.skipEncoding.get()) return rexNodeJson; // Write bytes of all serializable contents ByteArrayOutputStream output = new ByteArrayOutputStream(); ObjectOutputStream objectOutput = new ObjectOutputStream(output); - objectOutput.writeObject(envelope); + objectOutput.writeObject(rexNodeJson); objectOutput.flush(); - return CalcitePlanContext.skipEncoding.get() - ? rexNodeJson - : Base64.getEncoder().encodeToString(output.toByteArray()); + return Base64.getEncoder().encodeToString(output.toByteArray()); } catch (Exception e) { - throw new IllegalStateException( - "Failed to serialize RexNode: " + remappedRexInfo.getKey(), e); + throw new IllegalStateException("Failed to serialize RexNode: " + standardizedRexExpr, e); } } @@ -138,34 +115,23 @@ public String serialize(RexNode rexNode, RelDataType rowType, Map deserialize(String struct) { - Map objectMap = null; + public RexNode deserialize(String struct) { + String exprStr = null; try { - // Recover Map object from bytes ByteArrayInputStream input = new ByteArrayInputStream(Base64.getDecoder().decode(struct)); ObjectInputStream objectInput = new ObjectInputStream(input); - objectMap = (Map) objectInput.readObject(); + exprStr = (String) objectInput.readObject(); - // PPL Expr types are all serializable - Map fieldTypes = (Map) objectMap.get(FIELD_TYPES); // Deserialize RelDataType and RexNode by JSON RelJson relJson = ExtendedRelJson.create((JsonBuilder) null); - Map rowTypeMap = mapper.readValue((String) objectMap.get(ROW_TYPE), TYPE_REF); - RelDataType rowType = relJson.toType(cluster.getTypeFactory(), rowTypeMap); - OpenSearchRelInputTranslator inputTranslator = new OpenSearchRelInputTranslator(rowType); relJson = - relJson.withInputTranslator(inputTranslator).withOperatorTable(getPplSqlOperatorTable()); - Map exprMap = mapper.readValue((String) objectMap.get(EXPR), TYPE_REF); - RexNode rexNode = relJson.toRex(cluster, exprMap); - - return Map.of(EXPR, rexNode, FIELD_TYPES, fieldTypes, ROW_TYPE, rowType); + relJson + .withInputTranslator(ExtendedRelJson::translateInput) + .withOperatorTable(getPplSqlOperatorTable()); + Map exprMap = mapper.readValue(exprStr, TYPE_REF); + return relJson.toRex(cluster, exprMap); } catch (Exception e) { - if (objectMap == null) { - throw new IllegalStateException( - "Failed to deserialize RexNode due to object map is null", e); - } - throw new IllegalStateException( - "Failed to deserialize RexNode and its required structure: " + objectMap.get(EXPR), e); + throw new IllegalStateException("Failed to deserialize RexNode " + exprStr, e); } } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RexStandardizer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RexStandardizer.java new file mode 100644 index 00000000000..cbf10109d10 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RexStandardizer.java @@ -0,0 +1,194 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.serde; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.ConstantExpression; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBiVisitorImpl; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexCorrelVariable; +import org.apache.calcite.rex.RexDynamicParam; +import org.apache.calcite.rex.RexFieldAccess; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLambda; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexLocalRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexNodeAndFieldIndex; +import org.apache.calcite.rex.RexOver; +import org.apache.calcite.rex.RexPatternFieldRef; +import org.apache.calcite.rex.RexRangeRef; +import org.apache.calcite.rex.RexSubQuery; +import org.apache.calcite.rex.RexTableInputRef; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; +import org.opensearch.sql.opensearch.storage.script.CalciteScriptEngine.Source; + +/** + * This help standardizes the RexNode expression, the process including: 1. Replace RexInputRef with + * RexDynamicParam 2. Replace RexLiteral with RexDynamicParam TODO: 3. Replace RexCall with + * equivalent functions 4. Replace RelDataType with a wider type 5. Do constant folding + */ +public class RexStandardizer extends RexBiVisitorImpl { + private static final RexStandardizer standardizer = new RexStandardizer(true); + + protected RexStandardizer(boolean deep) { + super(deep); + } + + @Override + public RexNode visitCall(final RexCall call, ScriptParameterHelper helper) { + boolean[] update = {false}; + List clonedOperands = visitList(call.operands, helper, update); + if (update[0]) { + // REVIEW jvs 8-Mar-2005: This doesn't take into account + // the fact that a rewrite may have changed the result type. + // To do that, we would need to take a RexBuilder and + // watch out for special operators like CAST and NEW where + // the type is embedded in the original call. + return call.clone(call.getType(), clonedOperands); + } else { + return call; + } + } + + @Override + public RexNode visitInputRef(RexInputRef inputRef, ScriptParameterHelper helper) { + int index = inputRef.getIndex(); + RelDataTypeField field = helper.inputFieldList.get(index); + ExprType exprType = helper.fieldTypes.get(field.getName()); + String docFieldName = + exprType == ExprCoreType.STRUCT || exprType == ExprCoreType.ARRAY + ? null + : OpenSearchTextType.toKeywordSubField(field.getName(), exprType); + int newIndex = helper.sources.size(); + if (docFieldName != null) { + helper.digests.add(docFieldName); + helper.sources.add(Source.DOC_VALUE.getValue()); + } else { + helper.digests.add(field.getName()); + helper.sources.add(Source.SOURCE.getValue()); + } + return new RexDynamicParam(field.getType(), newIndex); + } + + @Override + public RexNode visitLiteral(RexLiteral literal, ScriptParameterHelper helper) { + /* + * 1. Skip replacing SARG/DECIMAL as it is not supported to translate literal; + * 2. Skip replacing SYMBOL/NULL as it may affect codegen, shouldn't be parameter; + * 3. Skip INTERVAL_TYPES as it has bug, introduced by calcite-1.41.1, TODO: remove this when fixed; + */ + if (literal.getTypeName() == SqlTypeName.SARG + || literal.getType().getSqlTypeName() == SqlTypeName.DECIMAL + || literal.getTypeName() == SqlTypeName.SYMBOL + || literal.getTypeName() == SqlTypeName.NULL + || SqlTypeName.INTERVAL_TYPES.contains(literal.getTypeName())) { + return literal; + } + + Object literalValue = translateLiteral(literal); + if (literalValue == null) return literal; + int newIndex = helper.sources.size(); + helper.sources.add(Source.LITERAL.getValue()); + helper.digests.add(literalValue); + return new RexDynamicParam(literal.getType(), newIndex); + } + + /** Override all below method to avoid returning null. */ + @Override + public RexNode visitLocalRef(RexLocalRef localRef, ScriptParameterHelper arg) { + return localRef; + } + + @Override + public RexNode visitOver(RexOver over, ScriptParameterHelper arg) { + return over; + } + + @Override + public RexNode visitCorrelVariable(RexCorrelVariable correlVariable, ScriptParameterHelper arg) { + return correlVariable; + } + + @Override + public RexNode visitDynamicParam(RexDynamicParam dynamicParam, ScriptParameterHelper arg) { + return dynamicParam; + } + + @Override + public RexNode visitRangeRef(RexRangeRef rangeRef, ScriptParameterHelper arg) { + return rangeRef; + } + + @Override + public RexNode visitFieldAccess(RexFieldAccess fieldAccess, ScriptParameterHelper arg) { + return fieldAccess; + } + + @Override + public RexNode visitSubQuery(RexSubQuery subQuery, ScriptParameterHelper arg) { + return subQuery; + } + + @Override + public RexNode visitTableInputRef(RexTableInputRef ref, ScriptParameterHelper arg) { + return ref; + } + + @Override + public RexNode visitPatternFieldRef(RexPatternFieldRef fieldRef, ScriptParameterHelper arg) { + return fieldRef; + } + + @Override + public RexNode visitLambda(RexLambda lambda, ScriptParameterHelper arg) { + return lambda; + } + + @Override + public RexNode visitNodeAndFieldIndex( + RexNodeAndFieldIndex nodeAndFieldIndex, ScriptParameterHelper arg) { + return nodeAndFieldIndex; + } + + protected List visitList( + List exprs, ScriptParameterHelper helper, boolean[] update) { + ImmutableList.Builder clonedOperands = ImmutableList.builder(); + for (RexNode operand : exprs) { + RexNode clonedOperand = operand.accept(this, helper); + if ((clonedOperand != operand) && (update != null)) { + update[0] = true; + } + clonedOperands.add(clonedOperand); + } + return clonedOperands.build(); + } + + private static Object translateLiteral(RexLiteral literal) { + org.apache.calcite.linq4j.tree.Expression expression = + RexToLixTranslator.translateLiteral( + literal, + literal.getType(), + OpenSearchTypeFactory.TYPE_FACTORY, + org.apache.calcite.adapter.enumerable.RexImpTable.NullAs.NOT_POSSIBLE); + if (expression instanceof ConstantExpression) { + return ((ConstantExpression) expression).value; + } + return null; + } + + public static RexNode standardizeRexNodeExpression( + RexNode rexNode, ScriptParameterHelper helper) { + return rexNode.accept(standardizer, helper); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ScriptParameterHelper.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ScriptParameterHelper.java new file mode 100644 index 00000000000..c30ba1c245d --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ScriptParameterHelper.java @@ -0,0 +1,86 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.serde; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import lombok.Getter; +import org.apache.calcite.DataContext.Variable; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.runtime.Hook; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.storage.script.CalciteScriptEngine.UnsupportedScriptException; + +@Getter +public class ScriptParameterHelper { + public static final String MISSING_MAX = "MISSING_MAX"; + + public static final String SOURCES = "SOURCES"; + public static final String DIGESTS = "DIGESTS"; + + /** Existing params derived from push down action, e.g., SORT_EXPR push down */ + Map existingParams; + + List inputFieldList; + Map fieldTypes; + + /** + * Records the source of each parameter, it decides which kind of source to retrieve value. + * + *

    0 stands for DOC_VALUE + * + *

    1 stand for SOURCE + * + *

    2 stands for LITERAL + */ + List sources; + + /** + * Records the digest of each parameter, which is used as the key to retrieve the value from the + * corresponding sources. It will be + * + *

    - field name for `DOC_VALUE` and `SOURCE`, + * + *

    - literal value itself for `LITERAL` + */ + List digests; + + public ScriptParameterHelper( + List inputFieldList, Map fieldTypes) { + this(inputFieldList, fieldTypes, Collections.emptyMap()); + } + + public ScriptParameterHelper( + List inputFieldList, + Map fieldTypes, + Map params) { + this.existingParams = params; + this.inputFieldList = inputFieldList; + this.fieldTypes = fieldTypes; + this.sources = new ArrayList<>(); + this.digests = new ArrayList<>(); + } + + public Map getParameters() { + // The timestamp when the query is executed, it's used by some time-related functions. + long currentTime = Hook.CURRENT_TIME.get(-1L); + if (currentTime < 0) { + throw new UnsupportedScriptException( + "ScriptQueryExpression requires a valid current time from hook, but it is not set"); + } + return new LinkedHashMap<>() { // Use LinkedHashMap to make the plan stable + { + putAll(existingParams); + put(Variable.UTC_TIMESTAMP.camelName, currentTime); + put(SOURCES, sources); + put(DIGESTS, digests); + } + }; + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java index 0f3841f2d85..87181e2535c 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/util/OpenSearchRelOptUtil.java @@ -27,49 +27,16 @@ import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexUtil; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.type.SqlTypeUtil; -import org.apache.calcite.util.mapping.Mapping; -import org.apache.calcite.util.mapping.Mappings; import org.apache.commons.lang3.tuple.Pair; -import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; import org.opensearch.sql.opensearch.storage.scan.context.PushDownType; import org.opensearch.sql.opensearch.storage.scan.context.SortExprDigest; @UtilityClass public class OpenSearchRelOptUtil { - private static final RemapIndexBiVisitor remapIndexBiVisitor = new RemapIndexBiVisitor(true); - - /** - * For pushed down RexNode, the input schema doesn't need to be the same with scan output schema - * because the input values are read from ScriptDocValues or source by field name. It doesn't - * matter what the actual index is. Current serialization will serialize map of field name and - * field ExprType, which could be a long serialized string. Use this method to narrow down input - * rowType and rewrite RexNode's input references. After that, we can leverage the fewer columns - * in the rowType to serialize least required field types. - * - * @param rexNode original RexNode to be pushed down - * @param inputRowType original input rowType of RexNode - * @return rewritten pair of RexNode and RelDataType - */ - public static Pair getRemappedRexAndType( - final RexNode rexNode, final RelDataType inputRowType) { - final BitSet seenOldIndex = new BitSet(); - final List newMappings = new ArrayList<>(); - rexNode.accept(remapIndexBiVisitor, Pair.of(seenOldIndex, newMappings)); - final List inputFieldList = inputRowType.getFieldList(); - final RelDataTypeFactory.Builder builder = OpenSearchTypeFactory.TYPE_FACTORY.builder(); - for (Integer oldIdx : newMappings) { - builder.add(inputFieldList.get(oldIdx)); - } - final Mapping mapping = Mappings.target(newMappings, inputRowType.getFieldCount()); - final RexNode newMappedRex = RexUtil.apply(mapping, rexNode); - return Pair.of(newMappedRex, builder.build()); - } - /** * Given an input Calcite RexNode, find the single input field with equivalent collation * information. The function returns the pair of input field index and a flag to indicate whether diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java index 62014bf5d28..13ea4c08535 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java @@ -490,10 +490,10 @@ void analyze_aggCall_complexScriptFilter() throws ExpressionNotAnalyzableExcepti "filter_complex_count")) .expectDslTemplate( "[{\"filter_bool_count\":{\"filter\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"*\\\"}\"," - + "\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":0}},\"boost\":1.0}}," + + "\"lang\":\"opensearch_compounded_script\",\"params\":{*}},\"boost\":1.0}}," + "\"aggregations\":{\"filter_bool_count\":{\"value_count\":{\"field\":\"_index\"}}}}}," + " {\"filter_complex_count\":{\"filter\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"*\\\"}\"," - + "\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":0}},\"boost\":1.0}}," + + "\"lang\":\"opensearch_compounded_script\",\"params\":{*}},\"boost\":1.0}}," + "\"aggregations\":{\"filter_complex_count\":{\"value_count\":{\"field\":\"_index\"}}}}}]") .expectResponseParser( new MetricParserHelper( diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJsonTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJsonTest.java index e4703db86c1..024b2068200 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJsonTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJsonTest.java @@ -302,13 +302,13 @@ void testDeserializeMapTypes() { "nullable", false, "key", - Map.of("udt", "EXPR_IP", "type", "VARCHAR", "nullable", false, "precision", -1), + Map.of("udt", "EXPR_IP", "type", "OTHER", "nullable", false, "precision", -1), "value", Map.of("udt", "EXPR_TIMESTAMP", "type", "VARCHAR", "nullable", false, "precision", -1)); RelDataType expectedComplexMap = typeFactory.createMapType( - typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_IP), + typeFactory.createSqlType(SqlTypeName.OTHER, false), typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP), false); @@ -316,8 +316,7 @@ void testDeserializeMapTypes() { assertEquals(expectedComplexMap, deserializedType); assertEquals( - typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_IP), - deserializedType.getKeyType()); + typeFactory.createSqlType(SqlTypeName.OTHER, false), deserializedType.getKeyType()); assertEquals( typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP), deserializedType.getValueType()); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializerTest.java index 1ba4d94c614..116ba5027fb 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializerTest.java @@ -7,7 +7,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.TYPE_FACTORY; +import java.util.List; import java.util.Map; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.volcano.VolcanoPlanner; @@ -28,12 +30,13 @@ import org.opensearch.sql.expression.function.PPLFuncImpTable; import org.opensearch.sql.opensearch.data.type.OpenSearchBinaryType; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType.MappingType; import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) public class RelJsonSerializerTest { - private final RexBuilder rexBuilder = new RexBuilder(OpenSearchTypeFactory.TYPE_FACTORY); + private final RexBuilder rexBuilder = new RexBuilder(TYPE_FACTORY); private final RelOptCluster cluster = RelOptCluster.create(new VolcanoPlanner(), rexBuilder); private final RelJsonSerializer serializer = new RelJsonSerializer(cluster); private final RelDataType rowType = @@ -52,13 +55,20 @@ void testSerializeAndDeserialize() { rexBuilder, BuiltinFunctionName.UPPER, rexBuilder.makeInputRef(rowType.getFieldList().get(0).getType(), 0)); + RexNode expectedNode = + PPLFuncImpTable.INSTANCE.resolve( + rexBuilder, + BuiltinFunctionName.UPPER, + rexBuilder.makeDynamicParam(rowType.getFieldList().get(0).getType(), 0)); - String code = serializer.serialize(rexUpper, rowType, fieldTypes); - Map objects = serializer.deserialize(code); + final ScriptParameterHelper helper = + new ScriptParameterHelper(rowType.getFieldList(), fieldTypes); + String code = serializer.serialize(rexUpper, helper); + RexNode rexNode = serializer.deserialize(code); - assertEquals(rexUpper, objects.get(RelJsonSerializer.EXPR)); - assertEquals(rowType, objects.get(RelJsonSerializer.ROW_TYPE)); - assertEquals(fieldTypes, objects.get(RelJsonSerializer.FIELD_TYPES)); + assertEquals(expectedNode, rexNode); + assertEquals(List.of(0), helper.sources); + assertEquals(List.of("Referer"), helper.digests); } @Test @@ -72,10 +82,7 @@ void testSerializeAndDeserializeUDT() { .add("time", UserDefinedFunctionUtils.NULLABLE_TIME_UDT) .add("timestamp", UserDefinedFunctionUtils.NULLABLE_TIMESTAMP_UDT) .add("ip", UserDefinedFunctionUtils.NULLABLE_IP_UDT) - .add( - "binary", - OpenSearchTypeFactory.TYPE_FACTORY.createUDT( - OpenSearchTypeFactory.ExprUDT.EXPR_BINARY)) + .add("binary", TYPE_FACTORY.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_BINARY)) .build(); Map fieldTypesWithUDT = Map.ofEntries( @@ -93,19 +100,32 @@ void testSerializeAndDeserializeUDT() { rexBuilder.makeInputRef(rowTypeWithUDT.getFieldList().get(2).getType(), 2), rexBuilder.makeInputRef(rowTypeWithUDT.getFieldList().get(3).getType(), 3), rexBuilder.makeInputRef(rowTypeWithUDT.getFieldList().get(4).getType(), 4)); - String serialized = serializer.serialize(rexNode, rowTypeWithUDT, fieldTypesWithUDT); - Map objects = serializer.deserialize(serialized); - assertEquals(rexNode, objects.get(RelJsonSerializer.EXPR)); - assertEquals(rowTypeWithUDT.toString(), objects.get(RelJsonSerializer.ROW_TYPE).toString()); - assertEquals(fieldTypesWithUDT, objects.get(RelJsonSerializer.FIELD_TYPES)); + RexNode expectedNode = + PPLFuncImpTable.INSTANCE.resolve( + rexBuilder, + BuiltinFunctionName.JSON_ARRAY, + rexBuilder.makeDynamicParam(rowTypeWithUDT.getFieldList().get(0).getType(), 0), + rexBuilder.makeDynamicParam(rowTypeWithUDT.getFieldList().get(1).getType(), 1), + rexBuilder.makeDynamicParam(rowTypeWithUDT.getFieldList().get(2).getType(), 2), + rexBuilder.makeDynamicParam(TYPE_FACTORY.createSqlType(SqlTypeName.OTHER, true), 3), + rexBuilder.makeDynamicParam(rowTypeWithUDT.getFieldList().get(4).getType(), 4)); + final ScriptParameterHelper helper = + new ScriptParameterHelper(rowTypeWithUDT.getFieldList(), fieldTypesWithUDT); + String serialized = serializer.serialize(rexNode, helper); + RexNode expr = serializer.deserialize(serialized); + assertEquals(expectedNode, expr); + assertEquals(List.of(0, 0, 0, 0, 0), helper.sources); + assertEquals(List.of("date", "time", "timestamp", "ip", "binary"), helper.digests); } @Test void testSerializeUnsupportedRexNode() { RexNode illegalRex = rexBuilder.makeRangeReference(rowType, 0, true); - assertThrows( - IllegalStateException.class, () -> serializer.serialize(illegalRex, rowType, fieldTypes)); + IllegalStateException.class, + () -> + serializer.serialize( + illegalRex, new ScriptParameterHelper(rowType.getFieldList(), fieldTypes))); } @Test @@ -121,8 +141,9 @@ void testDeserializeFunctionOutOfScope() { rexBuilder.makeInputRef(rowType.getFieldList().get(0).getType(), 0), rexBuilder.makeLiteral( 1, rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER))); - - String code = serializer.serialize(outOfScopeRex, rowType, fieldTypes); + final ScriptParameterHelper helper = + new ScriptParameterHelper(rowType.getFieldList(), fieldTypes); + String code = serializer.serialize(outOfScopeRex, helper); assertThrows(IllegalStateException.class, () -> serializer.deserialize(code)); } @@ -148,12 +169,52 @@ void testSerializeIndexRemappedRexNode() { rexBuilder, BuiltinFunctionName.UPPER, rexBuilder.makeInputRef(rowType.getFieldList().get(0).getType(), 0)); + RexNode expectedNode = + PPLFuncImpTable.INSTANCE.resolve( + rexBuilder, + BuiltinFunctionName.UPPER, + rexBuilder.makeDynamicParam(rowType.getFieldList().get(0).getType(), 0)); + final ScriptParameterHelper helper = + new ScriptParameterHelper(originalRowType.getFieldList(), originalFieldTypes); + String code = serializer.serialize(originalRexUpper, helper); + RexNode rex = serializer.deserialize(code); + assertEquals(expectedNode, rex); + } - String code = serializer.serialize(originalRexUpper, originalRowType, originalFieldTypes); - Map objects = serializer.deserialize(code); - - assertEquals(remappedRexUpper, objects.get(RelJsonSerializer.EXPR)); - assertEquals(rowType, objects.get(RelJsonSerializer.ROW_TYPE)); - assertEquals(fieldTypes, objects.get(RelJsonSerializer.FIELD_TYPES)); + @Test + void testSerializeAndDeserializeLiteral() { + RelDataType rowTypeWithUDT = + rexBuilder + .getTypeFactory() + .builder() + .kind(StructKind.FULLY_QUALIFIED) + .add("date", UserDefinedFunctionUtils.NULLABLE_DATE_UDT) + .add("text", TYPE_FACTORY.createSqlType(SqlTypeName.VARCHAR, true)) + .build(); + Map fieldTypesWithUDT = + Map.ofEntries( + Map.entry("date", OpenSearchDateType.of(ExprCoreType.DATE)), + Map.entry("text", OpenSearchDataType.of(MappingType.Text))); + RexNode rexNode = + PPLFuncImpTable.INSTANCE.resolve( + rexBuilder, + BuiltinFunctionName.JSON_ARRAY, + rexBuilder.makeInputRef(rowTypeWithUDT.getFieldList().get(0).getType(), 0), + rexBuilder.makeInputRef(rowTypeWithUDT.getFieldList().get(1).getType(), 1), + rexBuilder.makeLiteral(1, TYPE_FACTORY.createSqlType(SqlTypeName.INTEGER, true))); + RexNode expectedNode = + PPLFuncImpTable.INSTANCE.resolve( + rexBuilder, + BuiltinFunctionName.JSON_ARRAY, + rexBuilder.makeDynamicParam(rowTypeWithUDT.getFieldList().get(0).getType(), 0), + rexBuilder.makeDynamicParam(rowTypeWithUDT.getFieldList().get(1).getType(), 1), + rexBuilder.makeDynamicParam(TYPE_FACTORY.createSqlType(SqlTypeName.INTEGER, false), 2)); + final ScriptParameterHelper helper = + new ScriptParameterHelper(rowTypeWithUDT.getFieldList(), fieldTypesWithUDT); + String serialized = serializer.serialize(rexNode, helper); + RexNode expr = serializer.deserialize(serialized); + assertEquals(expectedNode, expr); + assertEquals(List.of(0, 1, 2), helper.sources); + assertEquals(List.of("date", "text", 1), helper.digests); } }