diff --git a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java index 2cb459394a84c..1ef7e2642d2dc 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java +++ b/presto-main-base/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java @@ -14,10 +14,10 @@ package com.facebook.presto.sql.planner.optimizations; import com.facebook.presto.Session; -import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.AggregationNode.Aggregation; @@ -51,7 +51,6 @@ import static com.facebook.presto.common.function.OperatorType.EQUAL; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.spi.plan.ProjectNode.Locality.LOCAL; @@ -214,10 +213,10 @@ public PlanNode visitAggregation(AggregationNode node, RewriteContext extraProper return this; } + public HiveQueryRunnerBuilder setExtraCoordinatorProperties(Map extraCoordinatorProperties) + { + this.extraCoordinatorProperties.putAll(extraCoordinatorProperties); + return this; + } + public HiveQueryRunnerBuilder setHiveProperties(Map hiveProperties) { this.hiveProperties.putAll(hiveProperties); diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManager.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManager.java index 3e93d67c2bd88..f0283c380af66 100644 --- a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManager.java +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/typemanager/NativeTypeManager.java @@ -53,9 +53,11 @@ import static com.facebook.presto.common.type.StandardTypes.IPPREFIX; import static com.facebook.presto.common.type.StandardTypes.JSON; import static com.facebook.presto.common.type.StandardTypes.MAP; +import static com.facebook.presto.common.type.StandardTypes.QDIGEST; import static com.facebook.presto.common.type.StandardTypes.REAL; import static com.facebook.presto.common.type.StandardTypes.ROW; import static com.facebook.presto.common.type.StandardTypes.SMALLINT; +import static com.facebook.presto.common.type.StandardTypes.TDIGEST; import static com.facebook.presto.common.type.StandardTypes.TIMESTAMP; import static com.facebook.presto.common.type.StandardTypes.TIMESTAMP_WITH_TIME_ZONE; import static com.facebook.presto.common.type.StandardTypes.TINYINT; @@ -101,7 +103,9 @@ public class NativeTypeManager ARRAY, DECIMAL, MAP, + QDIGEST, ROW, + TDIGEST, FunctionType.NAME); private final TypeManager typeManager; diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/AbstractTestAggregationsNative.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/AbstractTestAggregationsNative.java new file mode 100644 index 0000000000000..50ba544f3a84a --- /dev/null +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/AbstractTestAggregationsNative.java @@ -0,0 +1,275 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativetests; + +import com.facebook.presto.tests.AbstractTestAggregations; +import org.testng.annotations.Test; + +import static java.lang.String.format; + +public abstract class AbstractTestAggregationsNative + extends AbstractTestAggregations +{ + private static final String QDIGEST_TYPE = "qdigest"; + + private String storageFormat; + private String approxDistinctUnsupportedSignatureError; + private String charTypeUnsupportedError; + private String mergeAggFunctionUnsupportedError; + private String tdigestAggFunctionUnsupportedError; + private String timeTypeUnsupportedError; + private String valueAtQuantileFunctionUnsupportedError; + + public void init(String storageFormat, boolean sidecarEnabled) + { + this.storageFormat = storageFormat; + if (sidecarEnabled) { + charTypeUnsupportedError = ".*Unknown type: char.*"; + timeTypeUnsupportedError = ".*Unknown type: time.*"; + approxDistinctUnsupportedSignatureError = ".*Unexpected parameters \\(timestamp with time zone.*\\) for function.*"; + mergeAggFunctionUnsupportedError = ".*Unexpected parameters \\(qdigest.* for function native.default.merge.*"; + tdigestAggFunctionUnsupportedError = ".*Function native.default.tdigest_agg not registered.*"; + valueAtQuantileFunctionUnsupportedError = ".*Unexpected parameters \\(qdigest.* for function native.default.value_at_quantile.*"; + } + else { + charTypeUnsupportedError = "Failed to parse type.*char"; + timeTypeUnsupportedError = "Failed to parse type.*time"; + approxDistinctUnsupportedSignatureError = ".*Aggregate function signature is not supported.*"; + mergeAggFunctionUnsupportedError = ".*Aggregate function signature is not supported: presto.default.merge.*"; + tdigestAggFunctionUnsupportedError = ".*Aggregate function not registered: presto.default.tdigest_agg.*"; + valueAtQuantileFunctionUnsupportedError = ".*Scalar function presto.default.value_at_quantile not registered with arguments: \\(QDIGEST.*"; + } + } + + /// `approx_distinct` aggregate function returns a different value for certain datatypes in Presto C++, see this + /// issue for more details: https://github.com/facebookincubator/velox/issues/9761. + /// `approx_distinct` does not support arguments of type `TIMESTAMP WITH TIME ZONE` in Presto C++, see this issue + /// for more details: https://github.com/prestodb/presto/issues/24815. + /// Presto C++ does not support datatypes `CHAR` and `TIME`, see: + /// https://github.com/prestodb/presto/blob/master/presto-docs/src/main/sphinx/presto_cpp/limitations.rst. + @Override + @Test + public void testApproximateCountDistinct() + { + // test NULL + assertQuery("SELECT approx_distinct(NULL)", "SELECT 0"); + assertQuery("SELECT approx_distinct(NULL, 0.023)", "SELECT 0"); + + // test date + String orderdate = storageFormat.equals("DWRF") ? "cast(orderdate as DATE)" : "orderdate"; + assertQuery(format("SELECT approx_distinct(%s) FROM orders", orderdate), "SELECT 2372"); + assertQuery(format("SELECT approx_distinct(%s, 0.023) FROM orders", orderdate), "SELECT 2372"); + + // test timestamp + assertQuery("SELECT approx_distinct(CAST(orderdate AS TIMESTAMP)) FROM orders", "SELECT 2347"); + assertQuery("SELECT approx_distinct(CAST(orderdate AS TIMESTAMP), 0.023) FROM orders", "SELECT 2347"); + + // test timestamp with time zone + assertQueryFails("SELECT approx_distinct(CAST(orderdate AS TIMESTAMP WITH TIME ZONE)) FROM orders", + approxDistinctUnsupportedSignatureError, true); + assertQueryFails("SELECT approx_distinct(CAST(orderdate AS TIMESTAMP WITH TIME ZONE), 0.023) FROM orders", + approxDistinctUnsupportedSignatureError, true); + + // test time + assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME)) FROM orders", timeTypeUnsupportedError, true); + assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME), 0.023) FROM orders", timeTypeUnsupportedError, true); + + // test time with time zone + assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME WITH TIME ZONE)) FROM orders", timeTypeUnsupportedError, true); + assertQueryFails("SELECT approx_distinct(CAST(from_unixtime(custkey) AS TIME WITH TIME ZONE), 0.023) FROM orders", timeTypeUnsupportedError, true); + + // test short decimal + assertQuery("SELECT approx_distinct(CAST(custkey AS DECIMAL(18, 0))) FROM orders", "SELECT 990"); + assertQuery("SELECT approx_distinct(CAST(custkey AS DECIMAL(18, 0)), 0.023) FROM orders", "SELECT 990"); + + // test long decimal + assertQuery("SELECT approx_distinct(CAST(custkey AS DECIMAL(25, 20))) FROM orders", "SELECT 1013"); + assertQuery("SELECT approx_distinct(CAST(custkey AS DECIMAL(25, 20)), 0.023) FROM orders", "SELECT 1013"); + + // test real + assertQuery("SELECT approx_distinct(CAST(custkey AS REAL)) FROM orders", "SELECT 982"); + assertQuery("SELECT approx_distinct(CAST(custkey AS REAL), 0.023) FROM orders", "SELECT 982"); + + // test bigint + assertQuery("SELECT approx_distinct(custkey) FROM orders", "SELECT 990"); + assertQuery("SELECT approx_distinct(custkey, 0.023) FROM orders", "SELECT 990"); + + // test integer + assertQuery("SELECT approx_distinct(CAST(custkey AS INTEGER)) FROM orders", "SELECT 1028"); + assertQuery("SELECT approx_distinct(CAST(custkey AS INTEGER), 0.023) FROM orders", "SELECT 1028"); + + // test smallint + assertQuery("SELECT approx_distinct(CAST(custkey AS SMALLINT)) FROM orders", "SELECT 1023"); + assertQuery("SELECT approx_distinct(CAST(custkey AS SMALLINT), 0.023) FROM orders", "SELECT 1023"); + + // test tinyint + assertQuery("SELECT approx_distinct(CAST((custkey % 128) AS TINYINT)) FROM orders", "SELECT 128"); + assertQuery("SELECT approx_distinct(CAST((custkey % 128) AS TINYINT), 0.023) FROM orders", "SELECT 128"); + + // test double + assertQuery("SELECT approx_distinct(CAST(custkey AS DOUBLE)) FROM orders", "SELECT 1014"); + assertQuery("SELECT approx_distinct(CAST(custkey AS DOUBLE), 0.023) FROM orders", "SELECT 1014"); + + // test varchar + assertQuery("SELECT approx_distinct(CAST(custkey AS VARCHAR)) FROM orders", "SELECT 1036"); + assertQuery("SELECT approx_distinct(CAST(custkey AS VARCHAR), 0.023) FROM orders", "SELECT 1036"); + + // test char + assertQueryFails("SELECT approx_distinct(CAST(CAST(custkey AS VARCHAR) AS CHAR(20))) FROM orders", charTypeUnsupportedError, true); + assertQueryFails("SELECT approx_distinct(CAST(CAST(custkey AS VARCHAR) AS CHAR(20)), 0.023) FROM orders", charTypeUnsupportedError, true); + + // test varbinary + assertQuery("SELECT approx_distinct(to_utf8(CAST(custkey AS VARCHAR))) FROM orders", "SELECT 1036"); + assertQuery("SELECT approx_distinct(to_utf8(CAST(custkey AS VARCHAR)), 0.023) FROM orders", "SELECT 1036"); + } + + /// `sum_data_size_for_stats` returns a different value for `Varchar` and `Varbinary` datatypes in Presto C++, see: + /// https://github.com/prestodb/presto/issues/20909. `CHAR` datatype is not supported in Presto C++, see issue: + /// https://github.com/prestodb/presto/issues/21332. + @Override + @Test + public void testSumDataSizeForStats() + { + // varchar + assertQuery("SELECT \"sum_data_size_for_stats\"(comment) FROM orders", "SELECT 787364"); + + // char + // Presto removes trailing whitespaces when casting to CHAR. + // Hard code the expected data size since there is no easy to way to compute it in H2. + assertQueryFails("SELECT \"sum_data_size_for_stats\"(CAST(comment AS CHAR(1000))) FROM orders", + charTypeUnsupportedError, true); + + // varbinary + assertQuery("SELECT \"sum_data_size_for_stats\"(CAST(comment AS VARBINARY)) FROM orders", "SELECT 787364"); + + // array + assertQuery("SELECT \"sum_data_size_for_stats\"(ARRAY[comment]) FROM orders", "SELECT 847364"); + assertQuery("SELECT \"sum_data_size_for_stats\"(ARRAY[comment, comment]) FROM orders", "SELECT 1634728"); + + // map + assertQuery("SELECT \"sum_data_size_for_stats\"(map(ARRAY[1], ARRAY[comment])) FROM orders", "SELECT 907364"); + assertQuery("SELECT \"sum_data_size_for_stats\"(map(ARRAY[1, 2], ARRAY[comment, comment])) FROM orders", "SELECT 1754728"); + + // row + assertQuery("SELECT \"sum_data_size_for_stats\"(ROW(comment)) FROM orders", "SELECT 847364"); + assertQuery("SELECT \"sum_data_size_for_stats\"(ROW(comment, comment)) FROM orders", "SELECT 1634728"); + } + + /// `max_data_size_for_stats` returns a different value for `Varchar` and `Varbinary` datatypes in Presto C++, see: + /// https://github.com/prestodb/presto/issues/20909. `CHAR` datatype is not supported in Presto C++, see issue: + /// https://github.com/prestodb/presto/issues/21332. + @Override + @Test + public void testMaxDataSizeForStats() + { + // varchar + assertQuery("SELECT \"max_data_size_for_stats\"(comment) FROM orders", "select 82"); + + // char + assertQueryFails("SELECT \"max_data_size_for_stats\"(CAST(comment AS CHAR(1000))) FROM orders", + charTypeUnsupportedError, true); + + // varbinary + assertQuery("SELECT \"max_data_size_for_stats\"(CAST(comment AS VARBINARY)) FROM orders", "select 82"); + + // max_data_size_for_stats is not needed for array, map and row + } + + /// Function `tdigest_agg` is not supported in Presto C++, see: https://github.com/prestodb/presto/issues/24811. + /// `qdigest` datatype is not supported in Presto C++, see: https://github.com/prestodb/presto/issues/24814. + @Override + @Test(dataProvider = "getType") + public void testStatisticalDigest(String type) + { + String errorMessage = type.equals(QDIGEST_TYPE) ? valueAtQuantileFunctionUnsupportedError : tdigestAggFunctionUnsupportedError; + + assertQueryFails(format("SELECT value_at_quantile(%s_agg(CAST(orderkey AS DOUBLE)), 0.5E0) > 0 FROM lineitem", type), + errorMessage, true); + assertQueryFails(format("SELECT value_at_quantile(%s_agg(CAST(quantity AS DOUBLE)), 0.5E0) > 0 FROM lineitem", type), + errorMessage, true); + assertQueryFails(format("SELECT value_at_quantile(%s_agg(CAST(quantity AS DOUBLE)), 0.5E0) > 0 FROM lineitem", type), + errorMessage, true); + assertQueryFails(format("SELECT value_at_quantile(%s_agg(CAST(orderkey AS DOUBLE), 2), 0.5E0) > 0 FROM lineitem", type), + errorMessage, true); + assertQueryFails(format("SELECT value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 3), 0.5E0) > 0 FROM lineitem", type), + errorMessage, true); + assertQueryFails(format("SELECT value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 4), 0.5E0) > 0 FROM lineitem", type), + errorMessage, true); + assertQueryFails(format("SELECT value_at_quantile(%s_agg(CAST(orderkey AS DOUBLE), 2, 0.0001E0), 0.5E0) > 0 FROM lineitem", type), + errorMessage, true); + assertQueryFails(format("SELECT value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 3, 0.0001E0), 0.5E0) > 0 FROM lineitem", type), + errorMessage, true); + assertQueryFails(format("SELECT value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 4, 0.0001E0), 0.5E0) > 0 FROM lineitem", type), + errorMessage, true); + } + + /// Function `tdigest_agg` is not supported in Presto C++, see: https://github.com/prestodb/presto/issues/24811. + /// `qdigest` datatype is not supported in Presto C++, see: https://github.com/prestodb/presto/issues/24814. + @Override + @Test(dataProvider = "getType") + public void testStatisticalDigestGroupBy(String type) + { + String errorMessage = type.equals(QDIGEST_TYPE) ? valueAtQuantileFunctionUnsupportedError : tdigestAggFunctionUnsupportedError; + + assertQueryFails(format("SELECT partkey, value_at_quantile(%s_agg(CAST(orderkey AS DOUBLE)), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + errorMessage, true); + assertQueryFails(format("SELECT partkey, value_at_quantile(%s_agg(CAST(quantity AS DOUBLE)), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + errorMessage, true); + assertQueryFails(format("SELECT partkey, value_at_quantile(%s_agg(CAST(quantity AS DOUBLE)), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + errorMessage, true); + assertQueryFails(format("SELECT partkey, value_at_quantile(%s_agg(CAST(orderkey AS DOUBLE), 2), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + errorMessage, true); + assertQueryFails(format("SELECT partkey, value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 3), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + errorMessage, true); + assertQueryFails(format("SELECT partkey, value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 4), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + errorMessage, true); + assertQueryFails(format("SELECT partkey, value_at_quantile(%s_agg(CAST(orderkey AS DOUBLE), 2, 0.0001E0), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + errorMessage, true); + assertQueryFails(format("SELECT partkey, value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 3, 0.0001E0), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + errorMessage, true); + assertQueryFails(format("SELECT partkey, value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 4, 0.0001E0), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + errorMessage, true); + } + + /// Function `tdigest_agg` is not supported in Presto C++, see: https://github.com/prestodb/presto/issues/24811. + /// `qdigest` datatype is not supported in Presto C++, see: https://github.com/prestodb/presto/issues/24814. + @Override + @Test(dataProvider = "getType") + public void testStatisticalDigestMerge(String type) + { + String errorMessage = type.equals(QDIGEST_TYPE) ? mergeAggFunctionUnsupportedError : tdigestAggFunctionUnsupportedError; + assertQueryFails(format("SELECT value_at_quantile(merge(%s), 0.5E0) > 0 FROM (SELECT partkey, %s_agg(CAST(orderkey AS DOUBLE)) as %s FROM lineitem GROUP BY partkey)", + type, + type, + type), + errorMessage); + } + + /// Aggregate function `merge` is not supported for `tdigest` type in Presto C++, see issue for more details: + /// https://github.com/prestodb/presto/issues/24813. `qdigest` datatype is not supported in Presto C++, see: + /// https://github.com/prestodb/presto/issues/24814. + @Override + @Test(dataProvider = "getType") + public void testStatisticalDigestMergeGroupBy(String type) + { + String errorMessage = type.equals(QDIGEST_TYPE) ? mergeAggFunctionUnsupportedError : tdigestAggFunctionUnsupportedError; + assertQueryFails(format("SELECT partkey, value_at_quantile(merge(%s), 0.5E0) > 0 " + + "FROM (SELECT partkey, suppkey, %s_agg(CAST(orderkey AS DOUBLE)) as %s FROM lineitem GROUP BY partkey, suppkey)" + + "GROUP BY partkey", + type, + type, + type), + errorMessage); + } +} diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestAggregations.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestAggregations.java new file mode 100644 index 0000000000000..89d31a9257b95 --- /dev/null +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestAggregations.java @@ -0,0 +1,49 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativetests; + +import com.facebook.presto.testing.QueryRunner; +import org.testng.annotations.BeforeClass; + +import static java.lang.Boolean.parseBoolean; + +public class TestAggregations + extends AbstractTestAggregationsNative +{ + private String storageFormat; + private boolean sidecarEnabled; + + @BeforeClass + @Override + public void init() + throws Exception + { + storageFormat = System.getProperty("storageFormat", "PARQUET"); + sidecarEnabled = parseBoolean(System.getProperty("sidecarEnabled", "true")); + super.init(storageFormat, sidecarEnabled); + super.init(); + } + + @Override + protected QueryRunner createQueryRunner() throws Exception + { + return NativeTestsUtils.createNativeQueryRunner(storageFormat, sidecarEnabled); + } + + @Override + protected void createTables() + { + NativeTestsUtils.createTables(storageFormat); + } +} diff --git a/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestOptimizeMixedDistinctAggregations.java b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestOptimizeMixedDistinctAggregations.java new file mode 100644 index 0000000000000..9ff17c46fc9ef --- /dev/null +++ b/presto-native-tests/src/test/java/com/facebook/presto/nativetests/TestOptimizeMixedDistinctAggregations.java @@ -0,0 +1,64 @@ + +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.nativetests; + +import com.facebook.presto.testing.QueryRunner; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.BeforeClass; + +import static com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils.nativeHiveQueryRunnerBuilder; +import static com.facebook.presto.sidecar.NativeSidecarPluginQueryRunnerUtils.setupNativeSidecarPlugin; +import static java.lang.Boolean.parseBoolean; + +public class TestOptimizeMixedDistinctAggregations + extends AbstractTestAggregationsNative +{ + private String storageFormat; + private boolean sidecarEnabled; + + @BeforeClass + @Override + public void init() + throws Exception + { + storageFormat = System.getProperty("storageFormat", "PARQUET"); + sidecarEnabled = parseBoolean(System.getProperty("sidecarEnabled", "true")); + super.init(storageFormat, sidecarEnabled); + super.init(); + } + + @Override + protected QueryRunner createQueryRunner() throws Exception + { + QueryRunner queryRunner = nativeHiveQueryRunnerBuilder() + .setStorageFormat(storageFormat) + .setAddStorageFormatToPath(true) + .setUseThrift(true) + .setCoordinatorSidecarEnabled(sidecarEnabled) + .setExtraCoordinatorProperties( + ImmutableMap.of("optimizer.optimize-mixed-distinct-aggregations", "true")) + .build(); + if (sidecarEnabled) { + setupNativeSidecarPlugin(queryRunner); + } + return queryRunner; + } + + @Override + protected void createTables() + { + NativeTestsUtils.createTables(storageFormat); + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/StandardFunctionResolution.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/StandardFunctionResolution.java index 7a35ef7a0e9a0..2d6f73cda84ee 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/StandardFunctionResolution.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/StandardFunctionResolution.java @@ -62,6 +62,8 @@ public interface StandardFunctionResolution boolean isCountFunction(FunctionHandle functionHandle); + boolean isCountIfFunction(FunctionHandle functionHandle); + FunctionHandle countFunction(); FunctionHandle countFunction(Type valueType); diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestAggregations.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestAggregations.java index 295952d923702..b3d761a2c22ec 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestAggregations.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestAggregations.java @@ -18,6 +18,8 @@ import com.facebook.presto.testing.MaterializedRow; import com.google.common.collect.ImmutableMap; import org.testng.annotations.DataProvider; +import org.testng.annotations.Optional; +import org.testng.annotations.Parameters; import org.testng.annotations.Test; import java.util.List; @@ -650,18 +652,23 @@ public void testGroupByNullIf() assertQuery("SELECT NULLIF(1, orderkey), count(*) FROM orders GROUP BY orderkey"); } + @Parameters("storageFormat") @Test - public void testGroupByExtract() + public void testGroupByExtract(@Optional("PARQUET") String storageFormat) { + // DWRF does not support date type. + String format = (System.getProperty("storageFormat") == null) ? storageFormat : System.getProperty("storageFormat"); + String orderdate = format.equals("DWRF") ? "cast(orderdate as DATE)" : "orderdate"; + // whole expression in group by - assertQuery("SELECT EXTRACT(YEAR FROM orderdate), count(*) FROM orders GROUP BY EXTRACT(YEAR FROM orderdate)"); + assertQuery(format("SELECT EXTRACT(YEAR FROM %s), count(*) FROM orders GROUP BY EXTRACT(YEAR FROM %s)", orderdate, orderdate)); assertQuery( - "SELECT EXTRACT(YEAR FROM orderdate), count(*) FROM orders GROUP BY 1", - "SELECT EXTRACT(YEAR FROM orderdate), count(*) FROM orders GROUP BY EXTRACT(YEAR FROM orderdate)"); + format("SELECT EXTRACT(YEAR FROM %s), count(*) FROM orders GROUP BY 1", orderdate), + format("SELECT EXTRACT(YEAR FROM %s), count(*) FROM orders GROUP BY EXTRACT(YEAR FROM %s)", orderdate, orderdate)); // argument in group by - assertQuery("SELECT EXTRACT(YEAR FROM orderdate), count(*) FROM orders GROUP BY orderdate"); + assertQuery(format("SELECT EXTRACT(YEAR FROM %s), count(*) FROM orders GROUP BY orderdate", orderdate)); } @Test @@ -1120,20 +1127,33 @@ public void testGroupingSetsAliasedGroupingColumns() "SELECT NULL, NULL, SUM(CAST(quantity AS BIGINT)) FROM lineitem"); } + @Parameters("storageFormat") @Test - public void testGroupingSetMixedExpressionAndColumn() + public void testGroupingSetMixedExpressionAndColumn(@Optional("PARQUET") String storageFormat) { - assertQuery("SELECT suppkey, month(shipdate), SUM(CAST(quantity AS BIGINT)) FROM lineitem GROUP BY month(shipdate), ROLLUP(suppkey)", - "SELECT suppkey, month(shipdate), SUM(CAST(quantity AS BIGINT)) FROM lineitem GROUP BY month(shipdate), suppkey UNION ALL " + - "SELECT NULL, month(shipdate), SUM(CAST(quantity AS BIGINT)) FROM lineitem GROUP BY month(shipdate)"); + // DWRF does not support date type. + String format = (System.getProperty("storageFormat") == null) ? storageFormat : System.getProperty("storageFormat"); + String shipdate = format.equals("DWRF") ? "cast(shipdate as DATE)" : "shipdate"; + + assertQuery(format("SELECT suppkey, month(%s), SUM(CAST(quantity AS BIGINT)) FROM lineitem GROUP BY month(%s), ROLLUP(suppkey)", + shipdate, shipdate), + format("SELECT suppkey, month(%s), SUM(CAST(quantity AS BIGINT)) FROM lineitem GROUP BY month(%s), suppkey UNION ALL " + + "SELECT NULL, month(%s), SUM(CAST(quantity AS BIGINT)) FROM lineitem GROUP BY month(%s)", + shipdate, shipdate, shipdate, shipdate)); } + @Parameters("storageFormat") @Test - public void testGroupingSetMixedExpressionAndOrdinal() + public void testGroupingSetMixedExpressionAndOrdinal(@Optional("PARQUET") String storageFormat) { - assertQuery("SELECT suppkey, month(shipdate), SUM(CAST(quantity AS BIGINT)) FROM lineitem GROUP BY 2, ROLLUP(suppkey)", - "SELECT suppkey, month(shipdate), SUM(CAST(quantity AS BIGINT)) FROM lineitem GROUP BY month(shipdate), suppkey UNION ALL " + - "SELECT NULL, month(shipdate), SUM(CAST(quantity AS BIGINT)) FROM lineitem GROUP BY month(shipdate)"); + // DWRF does not support date type. + String format = (System.getProperty("storageFormat") == null) ? storageFormat : System.getProperty("storageFormat"); + String shipdate = format.equals("DWRF") ? "cast(shipdate as DATE)" : "shipdate"; + + assertQuery(format("SELECT suppkey, month(%s), SUM(CAST(quantity AS BIGINT)) FROM lineitem GROUP BY 2, ROLLUP(suppkey)", shipdate), + format("SELECT suppkey, month(%s), SUM(CAST(quantity AS BIGINT)) FROM lineitem GROUP BY month(%s), suppkey UNION ALL " + + "SELECT NULL, month(%s), SUM(CAST(quantity AS BIGINT)) FROM lineitem GROUP BY month(%s)", + shipdate, shipdate, shipdate, shipdate)); } @Test diff --git a/presto-tests/src/test/java/com/facebook/presto/tests/TestOptimizeMixedDistinctAggregations.java b/presto-tests/src/test/java/com/facebook/presto/tests/TestOptimizeMixedDistinctAggregations.java index 91df43fd3fc5a..19b685a64e34c 100644 --- a/presto-tests/src/test/java/com/facebook/presto/tests/TestOptimizeMixedDistinctAggregations.java +++ b/presto-tests/src/test/java/com/facebook/presto/tests/TestOptimizeMixedDistinctAggregations.java @@ -28,12 +28,5 @@ protected QueryRunner createQueryRunner() .build(); } - @Override - public void testCountDistinct() - { - assertQuery("SELECT COUNT(DISTINCT custkey + 1) FROM orders", "SELECT COUNT(*) FROM (SELECT DISTINCT custkey + 1 FROM orders) t"); - assertQuery("SELECT COUNT(DISTINCT linenumber), COUNT(*) from lineitem where linenumber < 0"); - } - // TODO add dedicated test cases and remove `extends AbstractTestAggregation` }