diff --git a/pom.xml b/pom.xml index afcddad87b223..1caeefb376530 100644 --- a/pom.xml +++ b/pom.xml @@ -154,6 +154,7 @@ presto-client presto-parser presto-main-base + presto-main-tests presto-main presto-built-in-worker-function-tools presto-ml @@ -840,6 +841,12 @@ ${project.version} + + com.facebook.presto + presto-main-tests + ${project.version} + + com.facebook.presto presto-built-in-worker-function-tools diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index ff115fdceddc4..936679b234628 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -372,6 +372,12 @@ test + + com.facebook.presto + presto-main-tests + test + + com.facebook.presto presto-hive-metastore diff --git a/presto-i18n-functions/pom.xml b/presto-i18n-functions/pom.xml index a38dc951a3cec..d787cb101d161 100644 --- a/presto-i18n-functions/pom.xml +++ b/presto-i18n-functions/pom.xml @@ -90,6 +90,12 @@ test-jar test + + + com.facebook.presto + presto-main-tests + test + diff --git a/presto-iceberg/pom.xml b/presto-iceberg/pom.xml index e3fc939ba9180..c7a57d800cb67 100644 --- a/presto-iceberg/pom.xml +++ b/presto-iceberg/pom.xml @@ -581,6 +581,11 @@ presto-main-base test + + com.facebook.presto + presto-main-tests + test + com.facebook.presto presto-main diff --git a/presto-main-base/pom.xml b/presto-main-base/pom.xml index 0a1d5ca8b2e68..d9003a304ae95 100644 --- a/presto-main-base/pom.xml +++ b/presto-main-base/pom.xml @@ -83,6 +83,12 @@ presto-expressions + + com.facebook.presto + presto-main-tests + test + + com.facebook.airlift bootstrap diff --git a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java index 0644a6ebbdb68..e9b3e8641c885 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java +++ b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java @@ -30,6 +30,7 @@ import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.FunctionsConfig; import com.facebook.presto.sql.analyzer.SemanticErrorCode; +import com.facebook.presto.tests.operator.scalar.TestFunctions; import com.google.common.collect.ImmutableList; import io.airlift.slice.Slice; import org.intellij.lang.annotations.Language; @@ -52,6 +53,7 @@ import static org.testng.Assert.fail; public abstract class AbstractTestFunctions + implements TestFunctions { private static final double DELTA = 1e-5; @@ -113,7 +115,8 @@ public FunctionAndTypeManager getFunctionAndTypeManager() return functionAssertions.getFunctionAndTypeManager(); } - protected void assertFunction(String projection, Type expectedType, Object expected) + @Override + public void assertFunction(String projection, Type expectedType, Object expected) { functionAssertions.assertFunction(projection, expectedType, expected); } @@ -215,7 +218,8 @@ public void assertCachedInstanceHasBoundedRetainedSize(String projection) functionAssertions.assertCachedInstanceHasBoundedRetainedSize(projection); } - protected void assertNotSupported(String projection, String message) + @Override + public void assertNotSupported(String projection, String message) { try { functionAssertions.executeProjectionWithFullEngine(projection); diff --git a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestArrayExceptFunction.java b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestArrayExceptFunction.java index 96c2db4dabc30..61ab2058d9011 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestArrayExceptFunction.java +++ b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestArrayExceptFunction.java @@ -14,35 +14,21 @@ package com.facebook.presto.operator.scalar; import com.facebook.presto.common.type.ArrayType; -import com.facebook.presto.common.type.RowType; +import com.facebook.presto.tests.operator.scalar.AbstractTestArrayExcept; import com.google.common.collect.ImmutableList; import org.testng.annotations.Test; -import static com.facebook.presto.common.type.BigintType.BIGINT; -import static com.facebook.presto.common.type.BooleanType.BOOLEAN; -import static com.facebook.presto.common.type.DoubleType.DOUBLE; import static com.facebook.presto.common.type.IntegerType.INTEGER; import static com.facebook.presto.common.type.UnknownType.UNKNOWN; import static com.facebook.presto.common.type.VarcharType.VARCHAR; -import static com.facebook.presto.common.type.VarcharType.createVarcharType; -import static java.util.Arrays.asList; import static java.util.Collections.singletonList; public class TestArrayExceptFunction extends AbstractTestFunctions + implements AbstractTestArrayExcept { @Test - public void testBasic() - { - assertFunction("array_except(ARRAY[1, 5, 3], ARRAY[3])", new ArrayType(INTEGER), ImmutableList.of(1, 5)); - assertFunction("array_except(ARRAY[CAST(1 as BIGINT), 5, 3], ARRAY[5])", new ArrayType(BIGINT), ImmutableList.of(1L, 3L)); - assertFunction("array_except(ARRAY[CAST('x' as VARCHAR), 'y', 'z'], ARRAY['x'])", new ArrayType(VARCHAR), ImmutableList.of("y", "z")); - assertFunction("array_except(ARRAY[true, false, null], ARRAY[true])", new ArrayType(BOOLEAN), asList(false, null)); - assertFunction("array_except(ARRAY[1.1E0, 5.4E0, 3.9E0], ARRAY[5, 5.4E0])", new ArrayType(DOUBLE), ImmutableList.of(1.1, 3.9)); - } - - @Test - public void testEmpty() + void testEmpty() { assertFunction("array_except(ARRAY[], ARRAY[])", new ArrayType(UNKNOWN), ImmutableList.of()); assertFunction("array_except(ARRAY[], ARRAY[1, 3])", new ArrayType(INTEGER), ImmutableList.of()); @@ -59,40 +45,4 @@ public void testNull() assertFunction("array_except(ARRAY[], ARRAY[NULL])", new ArrayType(UNKNOWN), ImmutableList.of()); assertFunction("array_except(ARRAY[NULL], ARRAY[])", new ArrayType(UNKNOWN), singletonList(null)); } - - @Test - public void testDuplicates() - { - assertFunction("array_except(ARRAY[1, 5, 3, 5, 1], ARRAY[3])", new ArrayType(INTEGER), ImmutableList.of(1, 5)); - assertFunction("array_except(ARRAY[CAST(1 as BIGINT), 5, 5, 3, 3, 3, 1], ARRAY[3, 5])", new ArrayType(BIGINT), ImmutableList.of(1L)); - assertFunction("array_except(ARRAY[CAST('x' as VARCHAR), 'x', 'y', 'z'], ARRAY['x', 'y', 'x'])", new ArrayType(VARCHAR), ImmutableList.of("z")); - assertFunction("array_except(ARRAY[true, false, null, true, false, null], ARRAY[true, true, true])", new ArrayType(BOOLEAN), asList(false, null)); - } - - @Test - public void testIndeterminateRows() - { - // test unsupported - assertFunction( - "array_except(ARRAY[(123, 'abc'), (123, NULL)], ARRAY[(123, 'abc'), (123, NULL)])", - new ArrayType(RowType.anonymous(ImmutableList.of(INTEGER, createVarcharType(3)))), - ImmutableList.of()); - assertFunction( - "array_except(ARRAY[(NULL, 'abc'), (123, null), (123, 'abc')], ARRAY[(456, 'def'),(NULL, 'abc')])", - new ArrayType(RowType.anonymous(ImmutableList.of(INTEGER, createVarcharType(3)))), - ImmutableList.of(asList(123, null), asList(123, "abc"))); - } - - @Test - public void testIndeterminateArrays() - { - assertFunction( - "array_except(ARRAY[ARRAY[123, 456], ARRAY[123, NULL]], ARRAY[ARRAY[123, 456], ARRAY[123, NULL]])", - new ArrayType(new ArrayType(INTEGER)), - ImmutableList.of()); - assertFunction( - "array_except(ARRAY[ARRAY[NULL, 456], ARRAY[123, null], ARRAY[123, 456]], ARRAY[ARRAY[456, 456],ARRAY[NULL, 456]])", - new ArrayType(new ArrayType(INTEGER)), - ImmutableList.of(asList(123, null), asList(123, 456))); - } } diff --git a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestArraySortFunction.java b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestArraySortFunction.java index a0df75f03b29d..cdb888a4d8473 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestArraySortFunction.java +++ b/presto-main-base/src/test/java/com/facebook/presto/operator/scalar/TestArraySortFunction.java @@ -13,30 +13,10 @@ */ package com.facebook.presto.operator.scalar; -import com.facebook.presto.common.type.ArrayType; -import com.google.common.collect.ImmutableList; -import org.testng.annotations.Test; - -import static com.facebook.presto.common.type.BigintType.BIGINT; -import static com.facebook.presto.common.type.IntegerType.INTEGER; -import static com.facebook.presto.common.type.VarcharType.createVarcharType; -import static java.util.Arrays.asList; +import com.facebook.presto.tests.operator.scalar.AbstractTestArraySort; public class TestArraySortFunction extends AbstractTestFunctions + implements AbstractTestArraySort { - @Test - public void testArraySort() - { - assertFunction("array_sort(ARRAY [5, 20, null, 5, 3, 50]) ", new ArrayType(INTEGER), - asList(3, 5, 5, 20, 50, null)); - assertFunction("array_sort(array['x', 'a', 'a', 'a', 'a', 'm', 'j', 'p'])", - new ArrayType(createVarcharType(1)), ImmutableList.of("a", "a", "a", "a", "j", "m", "p", "x")); - assertFunction("array_sort(sequence(-4, 3))", new ArrayType(BIGINT), - asList(-4L, -3L, -2L, -1L, 0L, 1L, 2L, 3L)); - assertFunction("array_sort(reverse(sequence(-4, 3)))", new ArrayType(BIGINT), - asList(-4L, -3L, -2L, -1L, 0L, 1L, 2L, 3L)); - assertFunction("repeat(1,4)", new ArrayType(INTEGER), asList(1, 1, 1, 1)); - assertFunction("cast(array[] as array)", new ArrayType(INTEGER), asList()); - } } diff --git a/presto-main-tests/pom.xml b/presto-main-tests/pom.xml new file mode 100644 index 0000000000000..8d019ed356d75 --- /dev/null +++ b/presto-main-tests/pom.xml @@ -0,0 +1,91 @@ + + + 4.0.0 + + + com.facebook.presto + presto-root + 0.296-SNAPSHOT + + + presto-main-tests + presto-main-tests + Presto Main Tests + + + ${project.parent.basedir} + true + 4g + + + + + com.google.guava + guava + + + + com.google.inject + guice + test + + + + com.facebook.airlift + log-manager + test + + + + com.facebook.airlift + json + test + + + + com.facebook.airlift + jaxrs + test + + + + com.facebook.airlift + node + test + + + + com.facebook.airlift + http-server + test + + + + com.facebook.airlift + bootstrap + test + + + + com.facebook.airlift + stats + test + + + + com.facebook.presto + presto-common + + + + org.testng + testng + + + + com.facebook.airlift + units + test + + + diff --git a/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/AbstractTestArrayExcept.java b/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/AbstractTestArrayExcept.java new file mode 100644 index 0000000000000..c2ccbf8ecddbf --- /dev/null +++ b/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/AbstractTestArrayExcept.java @@ -0,0 +1,77 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.tests.operator.scalar; + +import com.facebook.presto.common.type.ArrayType; +import com.facebook.presto.common.type.RowType; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.Test; + +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.common.type.DoubleType.DOUBLE; +import static com.facebook.presto.common.type.IntegerType.INTEGER; +import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.common.type.VarcharType.createVarcharType; +import static java.util.Arrays.asList; + +public interface AbstractTestArrayExcept + extends TestFunctions +{ + @Test + default void testBasic() + { + assertFunction("array_except(ARRAY[1, 5, 3], ARRAY[3])", new ArrayType(INTEGER), ImmutableList.of(1, 5)); + assertFunction("array_except(ARRAY[CAST(1 as BIGINT), 5, 3], ARRAY[5])", new ArrayType(BIGINT), ImmutableList.of(1L, 3L)); + assertFunction("array_except(ARRAY[CAST('x' as VARCHAR), 'y', 'z'], ARRAY['x'])", new ArrayType(VARCHAR), ImmutableList.of("y", "z")); + assertFunction("array_except(ARRAY[true, false, null], ARRAY[true])", new ArrayType(BOOLEAN), asList(false, null)); + assertFunction("array_except(ARRAY[1.1E0, 5.4E0, 3.9E0], ARRAY[5, 5.4E0])", new ArrayType(DOUBLE), ImmutableList.of(1.1, 3.9)); + } + + @Test + default void testDuplicates() + { + assertFunction("array_except(ARRAY[1, 5, 3, 5, 1], ARRAY[3])", new ArrayType(INTEGER), ImmutableList.of(1, 5)); + assertFunction("array_except(ARRAY[CAST(1 as BIGINT), 5, 5, 3, 3, 3, 1], ARRAY[3, 5])", new ArrayType(BIGINT), ImmutableList.of(1L)); + assertFunction("array_except(ARRAY[CAST('x' as VARCHAR), 'x', 'y', 'z'], ARRAY['x', 'y', 'x'])", new ArrayType(VARCHAR), ImmutableList.of("z")); + assertFunction("array_except(ARRAY[true, false, null, true, false, null], ARRAY[true, true, true])", new ArrayType(BOOLEAN), asList(false, null)); + } + + @Test + default void testIndeterminateRows() + { + // test unsupported + assertFunction( + "array_except(ARRAY[(123, 'abc'), (123, NULL)], ARRAY[(123, 'abc'), (123, NULL)])", + new ArrayType(RowType.anonymous(ImmutableList.of(INTEGER, createVarcharType(3)))), + ImmutableList.of()); + assertFunction( + "array_except(ARRAY[(NULL, 'abc'), (123, null), (123, 'abc')], ARRAY[(456, 'def'),(NULL, 'abc')])", + new ArrayType(RowType.anonymous(ImmutableList.of(INTEGER, createVarcharType(3)))), + ImmutableList.of(asList(123, null), asList(123, "abc"))); + } + + @Test + default void testIndeterminateArrays() + { + assertFunction( + "array_except(ARRAY[ARRAY[123, 456], ARRAY[123, NULL]], ARRAY[ARRAY[123, 456], ARRAY[123, NULL]])", + new ArrayType(new ArrayType(INTEGER)), + ImmutableList.of()); + assertFunction( + "array_except(ARRAY[ARRAY[NULL, 456], ARRAY[123, null], ARRAY[123, 456]], ARRAY[ARRAY[456, 456],ARRAY[NULL, 456]])", + new ArrayType(new ArrayType(INTEGER)), + ImmutableList.of(asList(123, null), asList(123, 456))); + } +} diff --git a/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/AbstractTestArraySort.java b/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/AbstractTestArraySort.java new file mode 100644 index 0000000000000..9a3740d61ed27 --- /dev/null +++ b/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/AbstractTestArraySort.java @@ -0,0 +1,47 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.tests.operator.scalar; + +import com.facebook.presto.common.type.ArrayType; +import com.google.common.collect.ImmutableList; +import org.testng.annotations.Test; + +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.common.type.IntegerType.INTEGER; +import static com.facebook.presto.common.type.VarcharType.createVarcharType; +import static java.util.Arrays.asList; + +public interface AbstractTestArraySort + extends TestFunctions +{ + @Test + default void testArraySort() + { + assertFunction("array_sort(ARRAY [5, 20, null, 5, 3, 50]) ", new ArrayType(INTEGER), + asList(3, 5, 5, 20, 50, null)); + assertFunction("array_sort(sequence(-4, 3))", new ArrayType(BIGINT), + asList(-4L, -3L, -2L, -1L, 0L, 1L, 2L, 3L)); + assertFunction("array_sort(reverse(sequence(-4, 3)))", new ArrayType(BIGINT), + asList(-4L, -3L, -2L, -1L, 0L, 1L, 2L, 3L)); + assertFunction("repeat(1,4)", new ArrayType(INTEGER), asList(1, 1, 1, 1)); + assertFunction("cast(array[] as array)", new ArrayType(INTEGER), asList()); + } + + @Test + default void testArraySortVarchar() + { + assertFunction("array_sort(array['x', 'a', 'a', 'a', 'a', 'm', 'j', 'p'])", + new ArrayType(createVarcharType(1)), ImmutableList.of("a", "a", "a", "a", "j", "m", "p", "x")); + } +} diff --git a/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/TestFunctions.java b/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/TestFunctions.java new file mode 100644 index 0000000000000..e72644201ef3b --- /dev/null +++ b/presto-main-tests/src/main/java/com/facebook/presto/tests/operator/scalar/TestFunctions.java @@ -0,0 +1,30 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.tests.operator.scalar; + +import com.facebook.presto.common.type.Type; + +public interface TestFunctions +{ + /** + * Asserts that the projection, representing a SQL expression comprising a scalar function call, returns the + * expected value of the expected type. + */ + void assertFunction(String projection, Type expectedType, Object expected); + + /** + * Asserts that the projection is not supported and that it fails with the expected error message. + */ + void assertNotSupported(String projection, String message); +} diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 075effcff022a..4c16e618b748e 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -139,5 +139,11 @@ test-jar test + + + com.facebook.presto + presto-main-tests + test + diff --git a/presto-native-execution/pom.xml b/presto-native-execution/pom.xml index 9679bd8618699..9eeb4b10250e6 100644 --- a/presto-native-execution/pom.xml +++ b/presto-native-execution/pom.xml @@ -97,6 +97,12 @@ test + + com.facebook.presto + presto-main-tests + test + + com.facebook.presto presto-main diff --git a/presto-native-tests/pom.xml b/presto-native-tests/pom.xml index fb100f8995ca9..aa20ad93e333b 100644 --- a/presto-native-tests/pom.xml +++ b/presto-native-tests/pom.xml @@ -39,6 +39,12 @@ test + + com.facebook.presto + presto-main-tests + test + + com.facebook.presto presto-main @@ -212,6 +218,12 @@ ${project.version} test + + + org.jetbrains + annotations + provided + diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/AbstractTestNativeFunctions.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/AbstractTestNativeFunctions.java new file mode 100644 index 0000000000000..434a1414ea1f0 --- /dev/null +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/AbstractTestNativeFunctions.java @@ -0,0 +1,193 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativetests.operator.scalar; + +import com.facebook.presto.common.type.Type; +import com.facebook.presto.nativetests.NativeTestsUtils; +import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.AbstractTestQueryFramework; +import com.facebook.presto.tests.operator.scalar.TestFunctions; +import org.intellij.lang.annotations.Language; +import org.testng.annotations.BeforeClass; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.facebook.presto.testing.assertions.Assert.assertEquals; +import static com.facebook.presto.tests.QueryAssertions.assertExceptionMessage; +import static java.lang.Boolean.parseBoolean; +import static java.lang.String.format; +import static org.testng.Assert.fail; + +public abstract class AbstractTestNativeFunctions + extends AbstractTestQueryFramework + implements TestFunctions +{ + public boolean sidecarEnabled; + private String storageFormat; + + @BeforeClass + @Override + public void init() + throws Exception + { + storageFormat = System.getProperty("storageFormat", "PARQUET"); + sidecarEnabled = parseBoolean(System.getProperty("sidecarEnabled", "true")); + super.init(); + } + + @Override + protected QueryRunner createQueryRunner() throws Exception + { + return NativeTestsUtils.createNativeQueryRunner(storageFormat, sidecarEnabled); + } + + @Override + public void assertFunction(String projection, Type expectedType, Object expected) + { + String query = format("SELECT %s", projection); + @Language("SQL") String rewritten = rewrite(query); + MaterializedResult result = computeActual(rewritten); + assertEquals(result.getTypes().get(0), expectedType); + assertEquals(result.getMaterializedRows().get(0).getField(0), expected); + } + + @Override + public void assertNotSupported(String projection, @Language("RegExp") String message) + { + String query = format("SELECT %s", projection); + @Language("SQL") String rewritten = rewrite(query); + try { + computeActual(rewritten); + fail("expected exception"); + } + catch (RuntimeException ex) { + assertExceptionMessage(rewritten, ex, message, true); + } + } + + /** + * Rewrite SQL of the form 'select cast(arg as type)' to 'select cast(a as type) from (values (arg)) t(a)', and + * SQL of the form 'select function(arg1, arg2, ...)' to + * 'select function(a, b, ...) from (values (arg1, arg2, ...)) t(a, b, ...)'. + * This ensures that the function is not constant-folded on the coordinator and is evaluated on the native workers. + * Note that any arguments to the function will still be constant-folded if possible. For instance, consider the + * SQL 'select function(a, b(c), d(e(f)), g, ...)'. Arguments such as 'b(c)' and 'd(e(f))' in the rewritten SQL + * 'select function(p, q, r, s, ...) from (values (a, b(c), d(e(f)), g, ...)) t(p, q, r, s, ...)' can be + * constant-folded on the coordinator. The rewrite only ensures that the top-level function call at depth 0 is not + * evaluated on the coordinator. + */ + public static String rewrite(String sql) + { + String rewrittenCast = tryRewriteCast(sql); + if (rewrittenCast != null) { + return rewrittenCast; + } + + String rewrittenFunctionCall = tryRewriteFunctionCall(sql); + if (rewrittenFunctionCall != null) { + return rewrittenFunctionCall; + } + + throw new IllegalArgumentException("Sql must be of form: select cast(arg as type) or select function(arg1, arg2, ...)"); + } + + /** + * Helper function to rewrite SQL of the form 'select function(arg1, arg2, ...)' to equivalent SQL + * 'select function(a, b, ...) from (values (arg1, arg2, ...)) t(a, b, ...)'. + */ + private static String tryRewriteFunctionCall(String sql) + { + Pattern pattern = Pattern.compile( + "^select\\s+([a-zA-Z_][a-zA-Z0-9_]*)\\s*\\((.*)\\)\\s*$", + Pattern.CASE_INSENSITIVE | Pattern.DOTALL); + Matcher matcher = pattern.matcher(sql); + if (matcher.matches()) { + String functionName = matcher.group(1); + String argsString = matcher.group(2); + int depth = 0; + boolean inSingleQuote = false; + int numArgs = 1; + for (int i = 0; i < argsString.length(); i++) { + char c = argsString.charAt(i); + if (c == '\'') { + inSingleQuote = !inSingleQuote; + } + if (!inSingleQuote) { + if (c == '(' || c == '[') { + depth++; + } + else if (c == ')' || c == ']') { + depth--; + } + else if (c == ',' && depth == 0) { + numArgs++; + } + } + } + + List aliases = new ArrayList<>(); + for (int i = 0; i < numArgs; i++) { + aliases.add(String.valueOf((char) ('a' + i))); + } + String aliasesString = String.join(", ", aliases); + + return format("select %s(%s) from (values (%s)) t(%s)", functionName, aliasesString, argsString, aliasesString); + } + return null; + } + + /** + * Helper function to rewrite SQL of the form 'select cast(arg as type)' to equivalent SQL + * 'select cast(a as type) from (values (arg)) t(a)'. + */ + private static String tryRewriteCast(String sql) + { + Pattern castPattern = Pattern.compile( + "^select\\s+cast\\s*\\((.*)\\)\\s*$", + Pattern.CASE_INSENSITIVE | Pattern.DOTALL); + Matcher castMatcher = castPattern.matcher(sql); + if (castMatcher.matches()) { + String castExpression = castMatcher.group(1).trim(); + int depth = 0; + boolean inSingleQuote = false; + + for (int i = 0; i < castExpression.length(); i++) { + char c = castExpression.charAt(i); + if (c == '\'') { + inSingleQuote = !inSingleQuote; + } + if (!inSingleQuote) { + if (c == '(' || c == '[') { + depth++; + } + if (c == ')' || c == ']') { + depth--; + } + if (depth == 0 && i + 4 <= castExpression.length() && + castExpression.substring(i, i + 4).equalsIgnoreCase(" as ")) { + String arg = castExpression.substring(0, i).trim(); + String type = castExpression.substring(i + 4).trim(); + return format("select cast(a as %s) from (values (%s)) t(a)", type, arg); + } + } + } + throw new IllegalArgumentException("Could not parse cast expression: " + sql); + } + return null; + } +} diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/TestArrayExceptFunction.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/TestArrayExceptFunction.java new file mode 100644 index 0000000000000..b5b3e3111aff8 --- /dev/null +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/TestArrayExceptFunction.java @@ -0,0 +1,49 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativetests.operator.scalar; + +import com.facebook.presto.common.type.ArrayType; +import com.facebook.presto.tests.operator.scalar.AbstractTestArrayExcept; +import com.google.common.collect.ImmutableList; +import org.intellij.lang.annotations.Language; +import org.testng.annotations.Test; + +import static com.facebook.presto.common.type.IntegerType.INTEGER; +import static com.facebook.presto.common.type.VarcharType.VARCHAR; + +public class TestArrayExceptFunction + extends AbstractTestNativeFunctions + implements AbstractTestArrayExcept +{ + @Language("RegExp") private static final String unknownTypeError = ".*not a known type kind: UNKNOWN.*"; + + @Test + public void testEmpty() + { + assertNotSupported("array_except(ARRAY[], ARRAY[])", unknownTypeError); + assertFunction("array_except(ARRAY[], ARRAY[1, 3])", new ArrayType(INTEGER), ImmutableList.of()); + assertFunction("array_except(ARRAY[CAST('abc' as VARCHAR)], ARRAY[])", new ArrayType(VARCHAR), ImmutableList.of("abc")); + } + + @Test + public void testNull() + { + assertNotSupported("array_except(ARRAY[NULL], NULL)", unknownTypeError); + assertNotSupported("array_except(NULL, NULL)", unknownTypeError); + assertNotSupported("array_except(NULL, ARRAY[NULL])", unknownTypeError); + assertNotSupported("array_except(ARRAY[NULL], ARRAY[NULL])", unknownTypeError); + assertNotSupported("array_except(ARRAY[], ARRAY[NULL])", unknownTypeError); + assertNotSupported("array_except(ARRAY[NULL], ARRAY[])", unknownTypeError); + } +} diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/TestArraySortFunction.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/TestArraySortFunction.java new file mode 100644 index 0000000000000..399aa9d255cf7 --- /dev/null +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/operator/scalar/TestArraySortFunction.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativetests.operator.scalar; + +import com.facebook.presto.tests.operator.scalar.AbstractTestArraySort; + +public class TestArraySortFunction + extends AbstractTestNativeFunctions + implements AbstractTestArraySort +{ +} diff --git a/presto-pinot/pom.xml b/presto-pinot/pom.xml index fe1e4646e4740..92376d1ba60ab 100644 --- a/presto-pinot/pom.xml +++ b/presto-pinot/pom.xml @@ -130,6 +130,13 @@ test-jar test + + + ${project.groupId} + presto-main-tests + ${project.version} + test + diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 402f476635eaa..abf9805d216b4 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -100,6 +100,12 @@ test-jar test + + + com.facebook.presto + presto-main-tests + test + diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/QueryAssertions.java b/presto-tests/src/main/java/com/facebook/presto/tests/QueryAssertions.java index 3da7b6e5b6320..5ca5b4821d1fe 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/QueryAssertions.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/QueryAssertions.java @@ -408,7 +408,7 @@ protected static void assertQueryReturnsEmptyResult(QueryRunner queryRunner, Ses } } - private static void assertExceptionMessage(String sql, Exception exception, @Language("RegExp") String regex, boolean usePatternMatcher) + public static void assertExceptionMessage(String sql, Exception exception, @Language("RegExp") String regex, boolean usePatternMatcher) { if (usePatternMatcher) { Pattern p = Pattern.compile(regex, Pattern.MULTILINE);