-
Notifications
You must be signed in to change notification settings - Fork 213
Filter script pushdown with RelJson serialization in Calcite #3859
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 6 commits
1894875
13acaf5
b37dc92
1abf8aa
bc6ecac
cb3a2fe
700fc29
fc77e5e
aba4fed
2cf247d
3049c0c
85f8df0
18ab9bf
5b18cc4
e522bc0
1da69d1
ee67876
c18006a
ffd26a7
3ff2f50
2fee6ee
08cd417
fde5cf0
d2a95ea
1dd6752
573552b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,6 +9,7 @@ | |
| import static org.opensearch.sql.util.MatcherUtils.assertJsonEqualsIgnoreId; | ||
|
|
||
| import java.io.IOException; | ||
| import org.junit.Ignore; | ||
| import org.junit.jupiter.api.Test; | ||
| import org.opensearch.client.ResponseException; | ||
| import org.opensearch.sql.legacy.TestUtils; | ||
|
|
@@ -302,4 +303,34 @@ public void testPatternsBrainMethodWithAggPushDownExplain() throws IOException { | |
| "source=opensearch-sql_test_index_account" | ||
| + "| patterns email method=brain mode=aggregation")); | ||
| } | ||
|
|
||
| @Ignore("The serialized string is unstable because of function properties") | ||
| @Test | ||
| public void testFilterScriptPushDownExplain() throws Exception { | ||
| String expected = | ||
| isCalciteEnabled() | ||
| ? loadFromFile("expectedOutput/calcite/explain_filter_script_push.json") | ||
| : loadFromFile("expectedOutput/ppl/explain_filter_script_push.json"); | ||
|
|
||
| assertJsonEqualsIgnoreId( | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. can we avoid compare all function properties?
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It's a member of FunctionExpression in V2. It has a reference of current Instant object. So the generated bytes string is not stable. It can't be avoided now. |
||
| expected, | ||
| explainQueryToString( | ||
| "source=opensearch-sql_test_index_account | where firstname ='Amber' and age - 2 = 30 |" | ||
| + " fields firstname, age")); | ||
| } | ||
|
|
||
| @Ignore("The serialized string is unstable because of function properties") | ||
| @Test | ||
| public void testFilterFunctionScriptPushDownExplain() throws Exception { | ||
| String expected = | ||
| isCalciteEnabled() | ||
| ? loadFromFile("expectedOutput/calcite/explain_filter_function_script_push.json") | ||
| : loadFromFile("expectedOutput/ppl/explain_filter_function_script_push.json"); | ||
|
|
||
| assertJsonEqualsIgnoreId( | ||
| expected, | ||
| explainQueryToString( | ||
| "source=opensearch-sql_test_index_account | where length(firstname) = 5 and abs(age) =" | ||
| + " 32 and balance = 39225 | fields firstname, age")); | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| { | ||
| "calcite": { | ||
| "logical": "LogicalProject(firstname=[$1], age=[$8])\n LogicalFilter(condition=[AND(=(CHAR_LENGTH($1), 5), =(ABS($8), 32), =($3, 39225))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", | ||
| "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, balance, age], FILTER->AND(=(CHAR_LENGTH($0), 5), =(ABS($2), 32), =($1, 39225)), PROJECT->[firstname, age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"must\":[{\"script\":{\"script\":{\"source\":\"public Object[] apply(Object root0) {\\n final String input_value = ((org.apache.calcite.DataContext) root0).get(\\\"firstname.keyword\\\") == null ? null : ((org.apache.calcite.DataContext) root0).get(\\\"firstname.keyword\\\").toString();\\n final Integer method_call_value = input_value == null ? null : Integer.valueOf(org.apache.calcite.runtime.SqlFunctions.charLength(input_value));\\n return new Object[] {\\n method_call_value == null ? null : Boolean.valueOf(method_call_value.intValue() == 5)};\\n}\\n\\n\",\"lang\":\"opensearch_compounded_script\",\"options\":{\"engine_type\":\"calcite\"},\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},{\"script\":{\"script\":{\"source\":\"public Object[] apply(Object root0) {\\n final Long input_value = (Long) ((org.apache.calcite.DataContext) root0).get(\\\"age\\\");\\n final Long method_call_value = input_value == null ? null : Long.valueOf(org.apache.calcite.runtime.SqlFunctions.abs(input_value.longValue()));\\n return new Object[] {\\n method_call_value == null ? null : Boolean.valueOf(method_call_value.longValue() == 32L)};\\n}\\n\\n\",\"lang\":\"opensearch_compounded_script\",\"options\":{\"engine_type\":\"calcite\"},\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},{\"term\":{\"balance\":{\"value\":39225,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"firstname\",\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| { | ||
| "calcite": { | ||
| "logical": "LogicalProject(firstname=[$1], age=[$8])\n LogicalFilter(condition=[AND(=($1, 'Amber'), =(-($8, 2), 30))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", | ||
| "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, age], FILTER->AND(=($0, 'Amber'), =(-($1, 2), 30))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"must\":[{\"term\":{\"firstname.keyword\":{\"value\":\"Amber\",\"boost\":1.0}}},{\"script\":{\"script\":{\"source\":\"public Object[] apply(Object root0) {\\n final Long input_value = (Long) ((org.apache.calcite.DataContext) root0).get(\\\"age\\\");\\n final Long binary_call_value = input_value == null ? null : Long.valueOf(input_value.longValue() - 2L);\\n return new Object[] {\\n binary_call_value == null ? null : Boolean.valueOf(binary_call_value.longValue() == 30L)};\\n}\\n\\n\",\"lang\":\"opensearch_compounded_script\",\"options\":{\"engine_type\":\"calcite\"},\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"firstname\",\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" | ||
| } | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
please add the test:
source=opensearch-sql_test_index_account | where firstname != ''There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@songkant-aws can you add above test case in explain IT?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Added the above test. But the test itself is optimized as bool filter query before hitting script pushdown codepath. Ideally we should find a query that is chosen by optimizer. I'm checking if there is one.