[Backport 2.19-dev] Support script push down on text field

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java

@@ -69,6 +69,7 @@ public void supportPushDownSortMergeJoin() throws IOException {
   }
 
   // Only for Calcite
+  @Ignore("We've supported script push down on text field")
   @Test
   public void supportPartialPushDown() throws IOException {
     Assume.assumeTrue("This test is only for push down enabled", isPushdownEnabled());
@@ -82,6 +83,7 @@ public void supportPartialPushDown() throws IOException {
   }
 
   // Only for Calcite
+  @Ignore("We've supported script push down on text field")
   @Test
   public void supportPartialPushDown_NoPushIfAllFailed() throws IOException {
     Assume.assumeTrue("This test is only for push down enabled", isPushdownEnabled());
@@ -94,6 +96,7 @@ public void supportPartialPushDown_NoPushIfAllFailed() throws IOException {
     assertJsonEqualsIgnoreId(expected, result);
   }
 
+  @Ignore("We've supported script push down on text field")
   @Test
   public void supportPartialPushDownScript() throws IOException {
     Assume.assumeTrue("This test is only for push down enabled", isPushdownEnabled());
@@ -156,6 +159,18 @@ public void noPushDownForAggOnWindow() throws IOException {
     assertJsonEqualsIgnoreId(expected, result);
   }
 
+  // Only for Calcite
+  @Test
+  public void supportPushDownScriptOnTextField() throws IOException {
+    Assume.assumeTrue("This test is only for push down enabled", isPushdownEnabled());
+    String result =
+        explainQueryToString(
+            "explain source=opensearch-sql_test_index_account | where length(address) > 0 | eval"
+                + " address_length = length(address) | stats count() by address_length");
+    String expected = loadFromFile("expectedOutput/calcite/explain_script_push_on_text.json");
+    assertJsonEqualsIgnoreId(expected, result);
+  }
+
   /**
    * Executes the PPL query and returns the result as a string with windows-style line breaks
    * replaced with Unix-style ones.

integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.json

@@ -0,0 +1,6 @@
+{
+  "calcite": {
+    "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n  LogicalProject(count()=[$1], address_length=[$0])\n    LogicalAggregate(group=[{0}], count()=[COUNT()])\n      LogicalProject(address_length=[CHAR_LENGTH($2)])\n        LogicalFilter(condition=[>(CHAR_LENGTH($2), 0)])\n          CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n",
+    "physical": "EnumerableLimit(fetch=[10000])\n  EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], address_length=[$t0])\n    CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[address], SCRIPT->>(CHAR_LENGTH($0), 0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT())], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJhZGRyZXNzIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiPiIsCiAgICAia2luZCI6ICJHUkVBVEVSX1RIQU4iLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAgICAgImtpbmQiOiAiQ0hBUl9MRU5HVEgiLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgIm5hbWUiOiAiJDAiCiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAdhZGRyZXNzc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAAAeHh4\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},\"_source\":{\"includes\":[\"address\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"address_length\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJhZGRyZXNzIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AKZ7CiAgIm9wIjogewogICAgIm5hbWUiOiAiQ0hBUl9MRU5HVEgiLAogICAgImtpbmQiOiAiQ0hBUl9MRU5HVEgiLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n"
+  }
+}

integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json

@@ -1,7 +1,6 @@
 {
   "calcite": {
     "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n  LogicalProject(firstname=[$1], age=[$8], address=[$2])\n    LogicalFilter(condition=[AND(=($2, '671 Bristol Street'), =(-($8, 2), 30))])\n      CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n",
-    "physical": "EnumerableLimit(fetch=[10000])\n  EnumerableCalc(expr#0..2=[{inputs}], expr#3=['671 Bristol Street':VARCHAR], expr#4=[=($t1, $t3)], firstname=[$t0], age=[$t2], address=[$t1], $condition=[$t4])\n    CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, address, age], SCRIPT->=(-($2, 2), 30)], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"must\":[{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n  \\\\\\\"op\\\\\\\": {\\\\n    \\\\\\\"name\\\\\\\": \\\\\\\"=\\\\\\\",\\\\n    \\\\\\\"kind\\\\\\\": \\\\\\\"EQUALS\\\\\\\",\\\\n    \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n  },\\\\n  \\\\\\\"operands\\\\\\\": [\\\\n    {\\\\n      \\\\\\\"op\\\\\\\": {\\\\n        \\\\\\\"name\\\\\\\": \\\\\\\"-\\\\\\\",\\\\n        \\\\\\\"kind\\\\\\\": \\\\\\\"MINUS\\\\\\\",\\\\n        \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n      },\\\\n      \\\\\\\"operands\\\\\\\": [\\\\n        {\\\\n          \\\\\\\"input\\\\\\\": 2,\\\\n          \\\\\\\"name\\\\\\\": \\\\\\\"$2\\\\\\\"\\\\n        },\\\\n        {\\\\n          \\\\\\\"literal\\\\\\\": 2,\\\\n          \\\\\\\"type\\\\\\\": {\\\\n            \\\\\\\"type\\\\\\\": \\\\\\\"INTEGER\\\\\\\",\\\\n            \\\\\\\"nullable\\\\\\\": false\\\\n          }\\\\n        }\\\\n      ],\\\\n      \\\\\\\"type\\\\\\\": {\\\\n        \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n        \\\\\\\"nullable\\\\\\\": true\\\\n      }\\\\n    },\\\\n    {\\\\n      \\\\\\\"literal\\\\\\\": 30,\\\\n      \\\\\\\"type\\\\\\\": {\\\\n        \\\\\\\"type\\\\\\\": \\\\\\\"INTEGER\\\\\\\",\\\\n        \\\\\\\"nullable\\\\\\\": false\\\\n      }\\\\n    }\\\\n  ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"firstname\",\"address\",\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n",
-    "extended": "public org.apache.calcite.linq4j.Enumerable bind(final org.apache.calcite.DataContext root) {\n  final org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan v1stashed = (org.opensearch.sql.opensearch.storage.scan.CalciteEnumerableIndexScan) root.get(\"v1stashed\");\n  final org.apache.calcite.linq4j.Enumerable _inputEnumerable = v1stashed.scan();\n  final org.apache.calcite.linq4j.AbstractEnumerable child = new org.apache.calcite.linq4j.AbstractEnumerable(){\n    public org.apache.calcite.linq4j.Enumerator enumerator() {\n      return new org.apache.calcite.linq4j.Enumerator(){\n          public final org.apache.calcite.linq4j.Enumerator inputEnumerator = _inputEnumerable.enumerator();\n          public void reset() {\n            inputEnumerator.reset();\n          }\n\n          public boolean moveNext() {\n            while (inputEnumerator.moveNext()) {\n              final Object[] current = (Object[]) inputEnumerator.current();\n              final String input_value = current[1] == null ? null : current[1].toString();\n              final Boolean binary_call_value = input_value == null ? null : Boolean.valueOf(org.apache.calcite.runtime.SqlFunctions.eq(input_value, \"671 Bristol Street\"));\n              if (binary_call_value != null && org.apache.calcite.runtime.SqlFunctions.toBoolean(binary_call_value)) {\n                return true;\n              }\n            }\n            return false;\n          }\n\n          public void close() {\n            inputEnumerator.close();\n          }\n\n          public Object current() {\n            final Object[] current = (Object[]) inputEnumerator.current();\n            final Object input_value = current[0];\n            final Object input_value0 = current[2];\n            final Object input_value1 = current[1];\n            return new Object[] {\n                input_value,\n                input_value0,\n                input_value1};\n          }\n\n        };\n    }\n\n  };\n  return child.take(10000);\n}\n\n\npublic Class getElementType() {\n  return java.lang.Object[].class;\n}\n\n\n"
+    "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[SCRIPT->AND(=($2, '671 Bristol Street'), =(-($8, 2), 30)), PROJECT->[firstname, age, address], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"must\":[{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n  \\\\\\\"op\\\\\\\": {\\\\n    \\\\\\\"name\\\\\\\": \\\\\\\"=\\\\\\\",\\\\n    \\\\\\\"kind\\\\\\\": \\\\\\\"EQUALS\\\\\\\",\\\\n    \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n  },\\\\n  \\\\\\\"operands\\\\\\\": [\\\\n    {\\\\n      \\\\\\\"input\\\\\\\": 2,\\\\n      \\\\\\\"name\\\\\\\": \\\\\\\"$2\\\\\\\"\\\\n    },\\\\n    {\\\\n      \\\\\\\"literal\\\\\\\": \\\\\\\"671 Bristol Street\\\\\\\",\\\\n      \\\\\\\"type\\\\\\\": {\\\\n        \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n        \\\\\\\"nullable\\\\\\\": false,\\\\n        \\\\\\\"precision\\\\\\\": -1\\\\n      }\\\\n    }\\\\n  ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n  \\\\\\\"op\\\\\\\": {\\\\n    \\\\\\\"name\\\\\\\": \\\\\\\"=\\\\\\\",\\\\n    \\\\\\\"kind\\\\\\\": \\\\\\\"EQUALS\\\\\\\",\\\\n    \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n  },\\\\n  \\\\\\\"operands\\\\\\\": [\\\\n    {\\\\n      \\\\\\\"op\\\\\\\": {\\\\n        \\\\\\\"name\\\\\\\": \\\\\\\"-\\\\\\\",\\\\n        \\\\\\\"kind\\\\\\\": \\\\\\\"MINUS\\\\\\\",\\\\n        \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n      },\\\\n      \\\\\\\"operands\\\\\\\": [\\\\n        {\\\\n          \\\\\\\"input\\\\\\\": 8,\\\\n          \\\\\\\"name\\\\\\\": \\\\\\\"$8\\\\\\\"\\\\n        },\\\\n        {\\\\n          \\\\\\\"literal\\\\\\\": 2,\\\\n          \\\\\\\"type\\\\\\\": {\\\\n            \\\\\\\"type\\\\\\\": \\\\\\\"INTEGER\\\\\\\",\\\\n            \\\\\\\"nullable\\\\\\\": false\\\\n          }\\\\n        }\\\\n      ],\\\\n      \\\\\\\"type\\\\\\\": {\\\\n        \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n        \\\\\\\"nullable\\\\\\\": true\\\\n      }\\\\n    },\\\\n    {\\\\n      \\\\\\\"literal\\\\\\\": 30,\\\\n      \\\\\\\"type\\\\\\\": {\\\\n        \\\\\\\"type\\\\\\\": \\\\\\\"INTEGER\\\\\\\",\\\\n        \\\\\\\"nullable\\\\\\\": false\\\\n      }\\\\n    }\\\\n  ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"firstname\",\"age\",\"address\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n"
   }
 }