diff --git a/presto-benchmark/src/test/java/com/facebook/presto/benchmark/BenchmarkDoubleStatisticalDigestAggregationFunctions.java b/presto-benchmark/src/test/java/com/facebook/presto/benchmark/BenchmarkDoubleStatisticalDigestAggregationFunctions.java new file mode 100644 index 0000000000000..ca1bf39c39212 --- /dev/null +++ b/presto-benchmark/src/test/java/com/facebook/presto/benchmark/BenchmarkDoubleStatisticalDigestAggregationFunctions.java @@ -0,0 +1,108 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.benchmark; + +import com.facebook.presto.plugin.memory.MemoryConnectorFactory; +import com.facebook.presto.testing.LocalQueryRunner; +import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.tpch.TpchConnectorFactory; +import com.google.common.collect.ImmutableMap; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.io.IOException; + +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static java.lang.String.format; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.openjdk.jmh.annotations.Mode.AverageTime; + +@OutputTimeUnit(MILLISECONDS) +@BenchmarkMode(AverageTime) +@Fork(2) +@Warmup(iterations = 10) +@Measurement(iterations = 10) +public class BenchmarkDoubleStatisticalDigestAggregationFunctions +{ + @State(Scope.Thread) + public static class Context + { + @Param({"1", "2", "3"}) + int scalingFactor; + + private LocalQueryRunner queryRunner; + + public LocalQueryRunner getQueryRunner() + { + return queryRunner; + } + + @Setup + public void setUp() + throws IOException + { + queryRunner = new LocalQueryRunner(testSessionBuilder() + .setCatalog("memory") + .setSchema("default") + .build()); + queryRunner.createCatalog("tpch", new TpchConnectorFactory(), ImmutableMap.of()); + queryRunner.createCatalog("memory", new MemoryConnectorFactory(), ImmutableMap.of()); + + queryRunner.execute(format("CREATE TABLE memory.default.testingtable AS SELECT totalprice from tpch.sf%s.orders", scalingFactor)); + } + + @TearDown + public void tearDown() + { + queryRunner.close(); + } + } + + @Benchmark + public MaterializedResult benchmarkTDigestAggregation(Context context) + { + return context.getQueryRunner() + .execute("SELECT tdigest_agg(totalprice) FROM memory.default.testingtable"); + } + + @Benchmark + public MaterializedResult benchmarkQuantileDigestAggregation(Context context) + { + return context.getQueryRunner() + .execute("SELECT qdigest_agg(totalprice) FROM memory.default.testingtable"); + } + + public static void main(String[] args) + throws Exception + { + Options options = new OptionsBuilder() + .include(".*" + BenchmarkDoubleStatisticalDigestAggregationFunctions.class.getSimpleName() + ".*") + .build(); + + new Runner(options) + .run(); + } +} diff --git a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningInternalAggregateFunction.java b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningInternalAggregateFunction.java index 58ab1e44b99a8..3083db1253f46 100644 --- a/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningInternalAggregateFunction.java +++ b/presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningInternalAggregateFunction.java @@ -35,11 +35,12 @@ import static com.facebook.presto.plugin.geospatial.GeometryType.GEOMETRY_TYPE_NAME; import static com.facebook.presto.plugin.geospatial.SpatialPartitioningAggregateFunction.NAME; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; import static com.facebook.presto.spi.type.StandardTypes.INTEGER; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static java.lang.Math.toIntExact; -@AggregationFunction(value = NAME, decomposable = false, hidden = true) +@AggregationFunction(value = NAME, decomposable = false, visibility = HIDDEN) public class SpatialPartitioningInternalAggregateFunction { private static final int MAX_SAMPLE_COUNT = 1_000_000; diff --git a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java index f4147f72ca4c8..1dbb556f38109 100644 --- a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -141,6 +141,7 @@ public final class SystemSessionProperties public static final String AGGREGATION_PARTITIONING_MERGING_STRATEGY = "aggregation_partitioning_merging_strategy"; public static final String LIST_BUILT_IN_FUNCTIONS_ONLY = "list_built_in_functions_only"; public static final String PARTITIONING_PRECISION_STRATEGY = "partitioning_precision_strategy"; + public static final String EXPERIMENTAL_FUNCTIONS_ENABLED = "experimental_functions_enabled"; private final List> sessionProperties; @@ -704,7 +705,12 @@ public SystemSessionProperties( featuresConfig.getPartitioningPrecisionStrategy(), false, value -> PartitioningPrecisionStrategy.valueOf(((String) value).toUpperCase()), - PartitioningPrecisionStrategy::name)); + PartitioningPrecisionStrategy::name), + booleanProperty( + EXPERIMENTAL_FUNCTIONS_ENABLED, + "Enable listing of functions marked as experimental", + featuresConfig.isExperimentalFunctionsEnabled(), + false)); } public List> getSessionProperties() @@ -1195,4 +1201,9 @@ public static boolean isExactPartitioningPreferred(Session session) return session.getSystemProperty(PARTITIONING_PRECISION_STRATEGY, PartitioningPrecisionStrategy.class) == PartitioningPrecisionStrategy.PREFER_EXACT_PARTITIONING; } + + public static boolean isExperimentalFunctionsEnabled(Session session) + { + return session.getSystemProperty(EXPERIMENTAL_FUNCTIONS_ENABLED, Boolean.class); + } } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunctionNamespaceManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunctionNamespaceManager.java index 8c6419d5abe35..a9278133a7933 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunctionNamespaceManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunctionNamespaceManager.java @@ -51,6 +51,7 @@ import com.facebook.presto.operator.aggregation.MaxDataSizeForStats; import com.facebook.presto.operator.aggregation.MergeHyperLogLogAggregation; import com.facebook.presto.operator.aggregation.MergeQuantileDigestFunction; +import com.facebook.presto.operator.aggregation.MergeTDigestFunction; import com.facebook.presto.operator.aggregation.RealCorrelationAggregation; import com.facebook.presto.operator.aggregation.RealCovarianceAggregation; import com.facebook.presto.operator.aggregation.RealGeometricMeanAggregations; @@ -135,6 +136,7 @@ import com.facebook.presto.operator.scalar.SplitToMapFunction; import com.facebook.presto.operator.scalar.SplitToMultimapFunction; import com.facebook.presto.operator.scalar.StringFunctions; +import com.facebook.presto.operator.scalar.TDigestFunctions; import com.facebook.presto.operator.scalar.TryFunction; import com.facebook.presto.operator.scalar.TypeOfFunction; import com.facebook.presto.operator.scalar.UrlFunctions; @@ -168,6 +170,7 @@ import com.facebook.presto.spi.function.ScalarFunctionImplementation; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.function.SqlFunction; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.function.SqlInvokedFunction; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -192,6 +195,7 @@ import com.facebook.presto.type.QuantileDigestOperators; import com.facebook.presto.type.RealOperators; import com.facebook.presto.type.SmallintOperators; +import com.facebook.presto.type.TDigestOperators; import com.facebook.presto.type.TimeOperators; import com.facebook.presto.type.TimeWithTimeZoneOperators; import com.facebook.presto.type.TimestampOperators; @@ -240,6 +244,9 @@ import static com.facebook.presto.operator.aggregation.QuantileDigestAggregationFunction.QDIGEST_AGG_WITH_WEIGHT_AND_ERROR; import static com.facebook.presto.operator.aggregation.RealAverageAggregation.REAL_AVERAGE_AGGREGATION; import static com.facebook.presto.operator.aggregation.ReduceAggregationFunction.REDUCE_AGG; +import static com.facebook.presto.operator.aggregation.TDigestAggregationFunction.TDIGEST_AGG; +import static com.facebook.presto.operator.aggregation.TDigestAggregationFunction.TDIGEST_AGG_WITH_WEIGHT; +import static com.facebook.presto.operator.aggregation.TDigestAggregationFunction.TDIGEST_AGG_WITH_WEIGHT_AND_COMPRESSION; import static com.facebook.presto.operator.aggregation.minmaxby.MaxByAggregationFunction.MAX_BY; import static com.facebook.presto.operator.aggregation.minmaxby.MaxByNAggregationFunction.MAX_BY_N_AGGREGATION; import static com.facebook.presto.operator.aggregation.minmaxby.MinByAggregationFunction.MIN_BY; @@ -305,6 +312,7 @@ import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.spi.function.FunctionKind.WINDOW; import static com.facebook.presto.spi.function.OperatorType.tryGetOperatorType; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypeSignatures; import static com.facebook.presto.sql.planner.LiteralEncoder.MAGIC_LITERAL_FUNCTION_PREFIX; @@ -662,7 +670,11 @@ public BuiltInFunctionNamespaceManager( .aggregate(BuildSetDigestAggregation.class) .scalars(SetDigestFunctions.class) .scalars(SetDigestOperators.class) - .scalars(WilsonInterval.class); + .scalars(WilsonInterval.class) + .scalars(TDigestOperators.class) + .scalars(TDigestFunctions.class) + .functions(TDIGEST_AGG, TDIGEST_AGG_WITH_WEIGHT, TDIGEST_AGG_WITH_WEIGHT_AND_COMPRESSION) + .function(MergeTDigestFunction.MERGE); switch (featuresConfig.getRegexLibrary()) { case JONI: @@ -974,9 +986,9 @@ private static class MagicLiteralFunction } @Override - public boolean isHidden() + public final SqlFunctionVisibility getVisibility() { - return true; + return HIDDEN; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java index 0c93ea1b14a1f..a94e2404187e6 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java @@ -14,6 +14,7 @@ package com.facebook.presto.metadata; import com.facebook.presto.Session; +import com.facebook.presto.SystemSessionProperties; import com.facebook.presto.operator.aggregation.InternalAggregationFunction; import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; import com.facebook.presto.operator.window.WindowFunctionSupplier; @@ -71,6 +72,8 @@ import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_NOT_FOUND; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_USER_ERROR; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.EXPERIMENTAL; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypeSignatures; import static com.facebook.presto.sql.planner.LiteralEncoder.MAGIC_LITERAL_FUNCTION_PREFIX; @@ -179,7 +182,8 @@ public List listFunctions(Session session) return managers.stream() .flatMap(manager -> manager.listFunctions().stream()) - .filter(function -> !function.isHidden()) + .filter(function -> function.getVisibility() == PUBLIC || + (function.getVisibility() == EXPERIMENTAL && SystemSessionProperties.isExperimentalFunctionsEnabled(session))) .collect(toImmutableList()); } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/PolymorphicScalarFunction.java b/presto-main/src/main/java/com/facebook/presto/metadata/PolymorphicScalarFunction.java index c015f044c0279..05bb681f898f8 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/PolymorphicScalarFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/PolymorphicScalarFunction.java @@ -22,6 +22,7 @@ import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ReturnPlaceConvention; import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ScalarImplementationChoice; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.util.Reflection; @@ -45,7 +46,7 @@ class PolymorphicScalarFunction extends SqlScalarFunction { private final String description; - private final boolean hidden; + private final SqlFunctionVisibility visibility; private final boolean deterministic; private final boolean calledOnNullInput; private final List choices; @@ -53,7 +54,7 @@ class PolymorphicScalarFunction PolymorphicScalarFunction( Signature signature, String description, - boolean hidden, + SqlFunctionVisibility visibility, boolean deterministic, boolean calledOnNullInput, List choices) @@ -61,16 +62,16 @@ class PolymorphicScalarFunction super(signature); this.description = description; - this.hidden = hidden; + this.visibility = visibility; this.deterministic = deterministic; this.calledOnNullInput = calledOnNullInput; this.choices = requireNonNull(choices, "choices is null"); } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return hidden; + return visibility; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/PolymorphicScalarFunctionBuilder.java b/presto-main/src/main/java/com/facebook/presto/metadata/PolymorphicScalarFunctionBuilder.java index 75265f9a1c691..48297bf51fe26 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/PolymorphicScalarFunctionBuilder.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/PolymorphicScalarFunctionBuilder.java @@ -18,6 +18,7 @@ import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ReturnPlaceConvention; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableList; @@ -32,6 +33,8 @@ import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.BLOCK_AND_POSITION; import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -45,7 +48,7 @@ public final class PolymorphicScalarFunctionBuilder private final Optional operatorType; private Signature signature; private String description; - private Optional hidden = Optional.empty(); + private Optional visibility = Optional.empty(); private Boolean deterministic; private Boolean calledOnNullInput; private final List choices = new ArrayList<>(); @@ -65,7 +68,7 @@ public PolymorphicScalarFunctionBuilder(Class clazz, OperatorType operatorTyp public PolymorphicScalarFunctionBuilder signature(Signature signature) { this.signature = requireNonNull(signature, "signature is null"); - this.hidden = Optional.of(hidden.orElse(isOperator(signature))); + this.visibility = Optional.of(visibility.orElse(isOperator(signature) ? HIDDEN : PUBLIC)); return this; } @@ -75,9 +78,9 @@ public PolymorphicScalarFunctionBuilder description(String description) return this; } - public PolymorphicScalarFunctionBuilder hidden(boolean hidden) + public PolymorphicScalarFunctionBuilder visibility(SqlFunctionVisibility visibility) { - this.hidden = Optional.of(hidden); + this.visibility = Optional.of(visibility); return this; } @@ -109,7 +112,7 @@ public SqlScalarFunction build() return new PolymorphicScalarFunction( signature, description, - hidden.orElse(false), + visibility.orElse(PUBLIC), deterministic, operatorType.map(OperatorType::isCalledOnNullInput).orElse(calledOnNullInput), choices); diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/SqlAggregationFunction.java b/presto-main/src/main/java/com/facebook/presto/metadata/SqlAggregationFunction.java index 17c193344b72d..45d6bc6564309 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/SqlAggregationFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/SqlAggregationFunction.java @@ -19,6 +19,7 @@ import com.facebook.presto.spi.function.LongVariableConstraint; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.function.TypeVariableConstraint; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spi.type.TypeSignature; @@ -28,6 +29,7 @@ import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.spi.function.FunctionKind.AGGREGATE; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Objects.requireNonNull; @@ -36,7 +38,7 @@ public abstract class SqlAggregationFunction extends BuiltInFunction { private final Signature signature; - private final boolean hidden; + private final SqlFunctionVisibility visibility; public static List createFunctionByAnnotations(Class aggregationDefinition) { @@ -69,13 +71,13 @@ protected SqlAggregationFunction( List argumentTypes, FunctionKind kind) { - this(createSignature(name, typeVariableConstraints, longVariableConstraints, returnType, argumentTypes, kind), false); + this(createSignature(name, typeVariableConstraints, longVariableConstraints, returnType, argumentTypes, kind), PUBLIC); } - protected SqlAggregationFunction(Signature signature, boolean hidden) + protected SqlAggregationFunction(Signature signature, SqlFunctionVisibility visibility) { this.signature = requireNonNull(signature, "signature is null"); - this.hidden = hidden; + this.visibility = visibility; } private static Signature createSignature( @@ -109,9 +111,9 @@ public final Signature getSignature() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return hidden; + return visibility; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/SqlOperator.java b/presto-main/src/main/java/com/facebook/presto/metadata/SqlOperator.java index 6c0bc3f948890..3112ab3b2136a 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/SqlOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/SqlOperator.java @@ -17,11 +17,14 @@ import com.facebook.presto.spi.function.LongVariableConstraint; import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.function.TypeVariableConstraint; import com.facebook.presto.spi.type.TypeSignature; import java.util.List; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; + public abstract class SqlOperator extends SqlScalarFunction { @@ -42,9 +45,9 @@ protected SqlOperator(OperatorType operatorType, List ty } @Override - public final boolean isHidden() + public final SqlFunctionVisibility getVisibility() { - return true; + return HIDDEN; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AggregationFromAnnotationsParser.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AggregationFromAnnotationsParser.java index 41bca338aa350..3727e77e7d20e 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AggregationFromAnnotationsParser.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AggregationFromAnnotationsParser.java @@ -136,7 +136,7 @@ private static AggregationHeader parseHeader(AnnotatedElement aggregationDefinit parseDescription(aggregationDefinition), aggregationAnnotation.decomposable(), aggregationAnnotation.isOrderSensitive(), - aggregationAnnotation.hidden()); + aggregationAnnotation.visibility()); } private static List parseHeaders(AnnotatedElement aggregationDefinition, AnnotatedElement toParse) @@ -150,7 +150,7 @@ private static List parseHeaders(AnnotatedElement aggregation parseDescription(aggregationDefinition, toParse), aggregationAnnotation.decomposable(), aggregationAnnotation.isOrderSensitive(), - aggregationAnnotation.hidden())) + aggregationAnnotation.visibility())) .collect(toImmutableList()); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AggregationHeader.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AggregationHeader.java index 0f80155cfa97a..3dca904e817ce 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AggregationHeader.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AggregationHeader.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.operator.aggregation; +import com.facebook.presto.spi.function.SqlFunctionVisibility; + import java.util.Optional; import static java.util.Objects.requireNonNull; @@ -23,15 +25,15 @@ public class AggregationHeader private final Optional description; private final boolean decomposable; private final boolean orderSensitive; - private final boolean hidden; + private final SqlFunctionVisibility visibility; - public AggregationHeader(String name, Optional description, boolean decomposable, boolean orderSensitive, boolean hidden) + public AggregationHeader(String name, Optional description, boolean decomposable, boolean orderSensitive, SqlFunctionVisibility visibility) { this.name = requireNonNull(name, "name cannot be null"); this.description = requireNonNull(description, "description cannot be null"); this.decomposable = decomposable; this.orderSensitive = orderSensitive; - this.hidden = hidden; + this.visibility = visibility; } public String getName() @@ -54,8 +56,8 @@ public boolean isOrderSensitive() return orderSensitive; } - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return hidden; + return visibility; } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MaxDataSizeForStats.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MaxDataSizeForStats.java index 595ca260d377d..5ea4fcc94e255 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MaxDataSizeForStats.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MaxDataSizeForStats.java @@ -28,9 +28,10 @@ import com.facebook.presto.spi.type.BigintType; import com.facebook.presto.spi.type.StandardTypes; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; import static java.lang.Math.max; -@AggregationFunction(value = MaxDataSizeForStats.NAME, hidden = true) +@AggregationFunction(value = MaxDataSizeForStats.NAME, visibility = HIDDEN) public final class MaxDataSizeForStats { public static final String NAME = "$internal$max_data_size_for_stats"; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeQuantileDigestFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeQuantileDigestFunction.java index 6fc810bc15d70..a03efdd2e4eb0 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeQuantileDigestFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeQuantileDigestFunction.java @@ -14,58 +14,35 @@ package com.facebook.presto.operator.aggregation; import com.facebook.airlift.stats.QuantileDigest; -import com.facebook.presto.bytecode.DynamicClassLoader; -import com.facebook.presto.metadata.BoundVariables; -import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.metadata.SqlAggregationFunction; -import com.facebook.presto.operator.aggregation.state.QuantileDigestState; -import com.facebook.presto.operator.aggregation.state.QuantileDigestStateFactory; -import com.facebook.presto.operator.aggregation.state.QuantileDigestStateSerializer; +import com.facebook.presto.operator.aggregation.state.StatisticalDigestState; import com.facebook.presto.spi.block.Block; -import com.facebook.presto.spi.block.BlockBuilder; -import com.facebook.presto.spi.function.AggregationFunction; -import com.facebook.presto.spi.function.CombineFunction; -import com.facebook.presto.spi.function.InputFunction; -import com.facebook.presto.spi.type.QuantileDigestType; -import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; -import com.facebook.presto.spi.type.TypeManager; -import com.facebook.presto.spi.type.TypeSignatureParameter; -import com.google.common.collect.ImmutableList; import java.lang.invoke.MethodHandle; -import java.util.List; -import static com.facebook.presto.operator.aggregation.AggregationMetadata.AccumulatorStateDescriptor; -import static com.facebook.presto.operator.aggregation.AggregationMetadata.ParameterMetadata; -import static com.facebook.presto.operator.aggregation.AggregationMetadata.ParameterMetadata.ParameterType.BLOCK_INDEX; -import static com.facebook.presto.operator.aggregation.AggregationMetadata.ParameterMetadata.ParameterType.BLOCK_INPUT_CHANNEL; -import static com.facebook.presto.operator.aggregation.AggregationMetadata.ParameterMetadata.ParameterType.STATE; -import static com.facebook.presto.operator.aggregation.AggregationUtils.generateAggregationName; -import static com.facebook.presto.spi.function.Signature.comparableTypeParameter; -import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.util.MoreMath.nearlyEqual; +import static com.facebook.presto.operator.aggregation.StatisticalDigestFactory.createStatisticalQuantileDigest; +import static com.facebook.presto.operator.aggregation.state.StatisticalDigestStateFactory.createQuantileDigestFactory; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; +import static com.facebook.presto.spi.type.StandardTypes.QDIGEST; import static com.facebook.presto.util.Reflection.methodHandle; -import static com.google.common.base.Preconditions.checkArgument; -@AggregationFunction("merge") -public final class MergeQuantileDigestFunction - extends SqlAggregationFunction +public class MergeQuantileDigestFunction + extends MergeStatisticalDigestFunction { public static final MergeQuantileDigestFunction MERGE = new MergeQuantileDigestFunction(); public static final String NAME = "merge"; - private static final MethodHandle INPUT_FUNCTION = methodHandle(MergeQuantileDigestFunction.class, "input", Type.class, QuantileDigestState.class, Block.class, int.class); - private static final MethodHandle COMBINE_FUNCTION = methodHandle(MergeQuantileDigestFunction.class, "combine", QuantileDigestState.class, QuantileDigestState.class); - private static final MethodHandle OUTPUT_FUNCTION = methodHandle(MergeQuantileDigestFunction.class, "output", QuantileDigestStateSerializer.class, QuantileDigestState.class, BlockBuilder.class); - private static final double COMPARISON_EPSILON = 1E-6; - public MergeQuantileDigestFunction() + private static final MethodHandle INPUT_FUNCTION = methodHandle( + MergeQuantileDigestFunction.class, + "input", + Type.class, + StatisticalDigestState.class, + Block.class, + int.class); + + private MergeQuantileDigestFunction() { - super(NAME, - ImmutableList.of(comparableTypeParameter("T")), - ImmutableList.of(), - parseTypeSignature("qdigest(T)"), - ImmutableList.of(parseTypeSignature("qdigest(T)"))); + super(NAME, QDIGEST, createQuantileDigestFactory(), PUBLIC); } @Override @@ -74,80 +51,14 @@ public String getDescription() return "Merges the input quantile digests into a single quantile digest"; } - @Override - public InternalAggregationFunction specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) - { - Type valueType = boundVariables.getTypeVariable("T"); - QuantileDigestType outputType = (QuantileDigestType) typeManager.getParameterizedType(StandardTypes.QDIGEST, - ImmutableList.of(TypeSignatureParameter.of(valueType.getTypeSignature()))); - return generateAggregation(outputType); - } - - private static InternalAggregationFunction generateAggregation(QuantileDigestType type) - { - DynamicClassLoader classLoader = new DynamicClassLoader(MapAggregationFunction.class.getClassLoader()); - QuantileDigestStateSerializer stateSerializer = new QuantileDigestStateSerializer(); - Type intermediateType = stateSerializer.getSerializedType(); - - AggregationMetadata metadata = new AggregationMetadata( - generateAggregationName(NAME, type.getTypeSignature(), ImmutableList.of(type.getTypeSignature())), - createInputParameterMetadata(type), - INPUT_FUNCTION.bindTo(type), - COMBINE_FUNCTION, - OUTPUT_FUNCTION.bindTo(stateSerializer), - ImmutableList.of(new AccumulatorStateDescriptor( - QuantileDigestState.class, - stateSerializer, - new QuantileDigestStateFactory())), - type); - - GenericAccumulatorFactoryBinder factory = AccumulatorCompiler.generateAccumulatorFactoryBinder(metadata, classLoader); - return new InternalAggregationFunction(NAME, ImmutableList.of(type), ImmutableList.of(intermediateType), type, true, true, factory); - } - - private static List createInputParameterMetadata(Type valueType) + public static void input(Type type, StatisticalDigestState state, Block value, int index) { - return ImmutableList.of( - new ParameterMetadata(STATE), - new ParameterMetadata(BLOCK_INPUT_CHANNEL, valueType), - new ParameterMetadata(BLOCK_INDEX)); + merge(state, createStatisticalQuantileDigest(new QuantileDigest(type.getSlice(value, index)))); } - @InputFunction - public static void input(Type type, QuantileDigestState state, Block value, int index) - { - merge(state, new QuantileDigest(type.getSlice(value, index))); - } - - @CombineFunction - public static void combine(QuantileDigestState state, QuantileDigestState otherState) - { - merge(state, otherState.getQuantileDigest()); - } - - private static void merge(QuantileDigestState state, QuantileDigest input) - { - if (input == null) { - return; - } - QuantileDigest previous = state.getQuantileDigest(); - if (previous == null) { - state.setQuantileDigest(input); - state.addMemoryUsage(input.estimatedInMemorySizeInBytes()); - } - else { - checkArgument(nearlyEqual(previous.getMaxError(), input.getMaxError(), COMPARISON_EPSILON), - "Cannot merge qdigests with different accuracies (%s vs. %s)", state.getQuantileDigest().getMaxError(), input.getMaxError()); - checkArgument(nearlyEqual(previous.getAlpha(), input.getAlpha(), COMPARISON_EPSILON), - "Cannot merge qdigests with different alpha values (%s vs. %s)", state.getQuantileDigest().getAlpha(), input.getAlpha()); - state.addMemoryUsage(-previous.estimatedInMemorySizeInBytes()); - previous.merge(input); - state.addMemoryUsage(previous.estimatedInMemorySizeInBytes()); - } - } - - public static void output(QuantileDigestStateSerializer serializer, QuantileDigestState state, BlockBuilder out) + @Override + protected MethodHandle getInputFunction() { - serializer.serialize(state, out); + return INPUT_FUNCTION; } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeStatisticalDigestFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeStatisticalDigestFunction.java new file mode 100644 index 0000000000000..83fe34257caf5 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeStatisticalDigestFunction.java @@ -0,0 +1,148 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation; + +import com.facebook.presto.bytecode.DynamicClassLoader; +import com.facebook.presto.metadata.BoundVariables; +import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.metadata.SqlAggregationFunction; +import com.facebook.presto.operator.aggregation.state.StatisticalDigestState; +import com.facebook.presto.operator.aggregation.state.StatisticalDigestStateFactory; +import com.facebook.presto.operator.aggregation.state.StatisticalDigestStateSerializer; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.function.QualifiedFunctionName; +import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.spi.type.TypeSignatureParameter; +import com.google.common.collect.ImmutableList; + +import java.lang.invoke.MethodHandle; +import java.util.List; + +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.aggregation.AggregationMetadata.AccumulatorStateDescriptor; +import static com.facebook.presto.operator.aggregation.AggregationMetadata.ParameterMetadata; +import static com.facebook.presto.operator.aggregation.AggregationMetadata.ParameterMetadata.ParameterType.BLOCK_INDEX; +import static com.facebook.presto.operator.aggregation.AggregationMetadata.ParameterMetadata.ParameterType.BLOCK_INPUT_CHANNEL; +import static com.facebook.presto.operator.aggregation.AggregationMetadata.ParameterMetadata.ParameterType.STATE; +import static com.facebook.presto.operator.aggregation.AggregationUtils.generateAggregationName; +import static com.facebook.presto.spi.function.FunctionKind.AGGREGATE; +import static com.facebook.presto.spi.function.Signature.comparableTypeParameter; +import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; +import static com.facebook.presto.util.Reflection.methodHandle; + +public abstract class MergeStatisticalDigestFunction + extends SqlAggregationFunction +{ + public final String name; + public final String type; + public final StatisticalDigestStateFactory factory; + + private static final MethodHandle COMBINE_FUNCTION = methodHandle( + MergeStatisticalDigestFunction.class, + "combine", + StatisticalDigestState.class, + StatisticalDigestState.class); + private static final MethodHandle OUTPUT_FUNCTION = methodHandle( + MergeStatisticalDigestFunction.class, + "output", + StatisticalDigestStateSerializer.class, + StatisticalDigestState.class, + BlockBuilder.class); + + MergeStatisticalDigestFunction(String name, String type, StatisticalDigestStateFactory factory, SqlFunctionVisibility visibility) + { + super(new Signature( + QualifiedFunctionName.of(DEFAULT_NAMESPACE, name), + AGGREGATE, + ImmutableList.of(comparableTypeParameter("T")), + ImmutableList.of(), + parseTypeSignature(type + "(T)"), + ImmutableList.of(parseTypeSignature(type + "(T)")), + false), + visibility); + this.name = name; + this.type = type; + this.factory = factory; + } + + @Override + public InternalAggregationFunction specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + { + Type valueType = boundVariables.getTypeVariable("T"); + Type outputType = typeManager.getParameterizedType(type, + ImmutableList.of(TypeSignatureParameter.of(valueType.getTypeSignature()))); + return generateAggregation(outputType); + } + + private InternalAggregationFunction generateAggregation(Type type) + { + DynamicClassLoader classLoader = new DynamicClassLoader(MapAggregationFunction.class.getClassLoader()); + StatisticalDigestStateSerializer stateSerializer = new StatisticalDigestStateSerializer(); + Type intermediateType = stateSerializer.getSerializedType(); + AggregationMetadata metadata = new AggregationMetadata( + generateAggregationName(name, type.getTypeSignature(), ImmutableList.of(type.getTypeSignature())), + createInputParameterMetadata(type), + getInputFunction().bindTo(type), + COMBINE_FUNCTION, + OUTPUT_FUNCTION.bindTo(stateSerializer), + ImmutableList.of(new AccumulatorStateDescriptor( + StatisticalDigestState.class, + stateSerializer, + factory)), + type); + + GenericAccumulatorFactoryBinder factory = AccumulatorCompiler.generateAccumulatorFactoryBinder(metadata, classLoader); + return new InternalAggregationFunction(name, ImmutableList.of(type), ImmutableList.of(intermediateType), type, true, true, factory); + } + + private static List createInputParameterMetadata(Type valueType) + { + return ImmutableList.of( + new ParameterMetadata(STATE), + new ParameterMetadata(BLOCK_INPUT_CHANNEL, valueType), + new ParameterMetadata(BLOCK_INDEX)); + } + + protected abstract MethodHandle getInputFunction(); + + public static void combine(StatisticalDigestState state, StatisticalDigestState otherState) + { + merge(state, otherState.getStatisticalDigest()); + } + + protected static void merge(StatisticalDigestState state, StatisticalDigest input) + { + if (input == null) { + return; + } + StatisticalDigest previous = state.getStatisticalDigest(); + if (previous == null) { + state.setStatisticalDigest(input); + state.addMemoryUsage(input.estimatedInMemorySizeInBytes()); + } + else { + state.addMemoryUsage(-previous.estimatedInMemorySizeInBytes()); + previous.merge(input.getDigest()); + state.addMemoryUsage(previous.estimatedInMemorySizeInBytes()); + } + } + + public static void output(StatisticalDigestStateSerializer serializer, StatisticalDigestState state, BlockBuilder out) + { + serializer.serialize(state, out); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeTDigestFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeTDigestFunction.java new file mode 100644 index 0000000000000..caafff0695d69 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeTDigestFunction.java @@ -0,0 +1,57 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation; + +import com.facebook.presto.operator.aggregation.state.StatisticalDigestState; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.type.Type; + +import java.lang.invoke.MethodHandle; + +import static com.facebook.presto.operator.aggregation.StatisticalDigestFactory.createStatisticalTDigest; +import static com.facebook.presto.operator.aggregation.state.StatisticalDigestStateFactory.createTDigestFactory; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.EXPERIMENTAL; +import static com.facebook.presto.spi.type.StandardTypes.TDIGEST; +import static com.facebook.presto.tdigest.TDigest.createTDigest; +import static com.facebook.presto.util.Reflection.methodHandle; + +public class MergeTDigestFunction + extends MergeStatisticalDigestFunction +{ + public static final MergeTDigestFunction MERGE = new MergeTDigestFunction(); + public static final String NAME = "merge"; + private static final MethodHandle INPUT_FUNCTION = methodHandle(MergeTDigestFunction.class, "input", Type.class, StatisticalDigestState.class, Block.class, int.class); + + private MergeTDigestFunction() + { + super(NAME, TDIGEST, createTDigestFactory(), EXPERIMENTAL); + } + + @Override + public String getDescription() + { + return "Merges the input t-digests into a single t-digest"; + } + + public static void input(Type type, StatisticalDigestState state, Block value, int index) + { + merge(state, createStatisticalTDigest(createTDigest(type.getSlice(value, index)))); + } + + @Override + protected MethodHandle getInputFunction() + { + return INPUT_FUNCTION; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ParametricAggregation.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ParametricAggregation.java index 6d44fa1933779..95c6458283639 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ParametricAggregation.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/ParametricAggregation.java @@ -58,7 +58,7 @@ public ParametricAggregation( AggregationHeader details, ParametricImplementationsGroup implementations) { - super(signature, details.isHidden()); + super(signature, details.getVisibility()); this.details = requireNonNull(details, "details is null"); this.implementations = requireNonNull(implementations, "implementations is null"); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/QuantileDigestAggregationFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/QuantileDigestAggregationFunction.java index e14e5708df592..58f9759396021 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/QuantileDigestAggregationFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/QuantileDigestAggregationFunction.java @@ -14,214 +14,72 @@ package com.facebook.presto.operator.aggregation; import com.facebook.airlift.stats.QuantileDigest; -import com.facebook.presto.bytecode.DynamicClassLoader; -import com.facebook.presto.metadata.BoundVariables; -import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.metadata.SqlAggregationFunction; -import com.facebook.presto.operator.aggregation.state.QuantileDigestState; -import com.facebook.presto.operator.aggregation.state.QuantileDigestStateFactory; -import com.facebook.presto.operator.aggregation.state.QuantileDigestStateSerializer; -import com.facebook.presto.spi.block.BlockBuilder; -import com.facebook.presto.spi.type.QuantileDigestType; +import com.facebook.presto.operator.aggregation.state.StatisticalDigestState; import com.facebook.presto.spi.type.StandardTypes; -import com.facebook.presto.spi.type.Type; -import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spi.type.TypeSignature; -import com.facebook.presto.spi.type.TypeSignatureParameter; -import com.google.common.collect.ImmutableList; -import java.lang.invoke.MethodHandle; -import java.util.List; -import java.util.stream.Collectors; - -import static com.facebook.presto.operator.aggregation.AggregationMetadata.AccumulatorStateDescriptor; -import static com.facebook.presto.operator.aggregation.AggregationMetadata.ParameterMetadata; -import static com.facebook.presto.operator.aggregation.AggregationMetadata.ParameterMetadata.ParameterType.INPUT_CHANNEL; -import static com.facebook.presto.operator.aggregation.AggregationMetadata.ParameterMetadata.ParameterType.STATE; -import static com.facebook.presto.operator.aggregation.AggregationUtils.generateAggregationName; import static com.facebook.presto.operator.aggregation.FloatingPointBitsConverterUtil.doubleToSortableLong; import static com.facebook.presto.operator.aggregation.FloatingPointBitsConverterUtil.floatToSortableInt; -import static com.facebook.presto.operator.scalar.QuantileDigestFunctions.DEFAULT_ACCURACY; -import static com.facebook.presto.operator.scalar.QuantileDigestFunctions.DEFAULT_WEIGHT; +import static com.facebook.presto.operator.aggregation.StatisticalDigestFactory.createStatisticalQuantileDigest; +import static com.facebook.presto.operator.aggregation.state.StatisticalDigestStateFactory.createQuantileDigestFactory; import static com.facebook.presto.operator.scalar.QuantileDigestFunctions.verifyAccuracy; import static com.facebook.presto.operator.scalar.QuantileDigestFunctions.verifyWeight; -import static com.facebook.presto.spi.function.Signature.comparableTypeParameter; -import static com.facebook.presto.spi.type.BigintType.BIGINT; -import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; +import static com.facebook.presto.spi.type.StandardTypes.QDIGEST; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.util.Reflection.methodHandle; -import static com.google.common.collect.ImmutableList.toImmutableList; import static java.lang.Float.intBitsToFloat; -import static java.lang.String.format; -import static java.lang.invoke.MethodHandles.insertArguments; public final class QuantileDigestAggregationFunction - extends SqlAggregationFunction + extends StatisticalDigestAggregationFunction { public static final QuantileDigestAggregationFunction QDIGEST_AGG = new QuantileDigestAggregationFunction(parseTypeSignature("V")); - public static final QuantileDigestAggregationFunction QDIGEST_AGG_WITH_WEIGHT = new QuantileDigestAggregationFunction(parseTypeSignature("V"), parseTypeSignature(StandardTypes.BIGINT)); - public static final QuantileDigestAggregationFunction QDIGEST_AGG_WITH_WEIGHT_AND_ERROR = new QuantileDigestAggregationFunction(parseTypeSignature("V"), parseTypeSignature(StandardTypes.BIGINT), parseTypeSignature(StandardTypes.DOUBLE)); - public static final String NAME = "qdigest_agg"; + public static final QuantileDigestAggregationFunction QDIGEST_AGG_WITH_WEIGHT = new QuantileDigestAggregationFunction( + parseTypeSignature("V"), + parseTypeSignature(StandardTypes.BIGINT)); + public static final QuantileDigestAggregationFunction QDIGEST_AGG_WITH_WEIGHT_AND_ERROR = new QuantileDigestAggregationFunction( + parseTypeSignature("V"), + parseTypeSignature(StandardTypes.BIGINT), + parseTypeSignature(StandardTypes.DOUBLE)); - private static final MethodHandle INPUT_DOUBLE = methodHandle(QuantileDigestAggregationFunction.class, "inputDouble", QuantileDigestState.class, double.class, long.class, double.class); - private static final MethodHandle INPUT_REAL = methodHandle(QuantileDigestAggregationFunction.class, "inputReal", QuantileDigestState.class, long.class, long.class, double.class); - private static final MethodHandle INPUT_BIGINT = methodHandle(QuantileDigestAggregationFunction.class, "inputBigint", QuantileDigestState.class, long.class, long.class, double.class); - private static final MethodHandle COMBINE_FUNCTION = methodHandle(QuantileDigestAggregationFunction.class, "combineState", QuantileDigestState.class, QuantileDigestState.class); - private static final MethodHandle OUTPUT_FUNCTION = methodHandle(QuantileDigestAggregationFunction.class, "evaluateFinal", QuantileDigestStateSerializer.class, QuantileDigestState.class, BlockBuilder.class); + public static final String NAME = "qdigest_agg"; private QuantileDigestAggregationFunction(TypeSignature... typeSignatures) { - super( - NAME, - ImmutableList.of(comparableTypeParameter("V")), - ImmutableList.of(), - parseTypeSignature("qdigest(V)"), - ImmutableList.copyOf(typeSignatures)); + super(NAME, QDIGEST, createQuantileDigestFactory(), PUBLIC, typeSignatures); } @Override public String getDescription() { - return "Returns a qdigest from the set of reals, bigints or doubles"; - } - - @Override - public InternalAggregationFunction specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) - { - Type valueType = boundVariables.getTypeVariable("V"); - QuantileDigestType outputType = (QuantileDigestType) typeManager.getParameterizedType( - StandardTypes.QDIGEST, - ImmutableList.of(TypeSignatureParameter.of(valueType.getTypeSignature()))); - return generateAggregation(valueType, outputType, arity); - } - - private static InternalAggregationFunction generateAggregation(Type valueType, QuantileDigestType outputType, int arity) - { - DynamicClassLoader classLoader = new DynamicClassLoader(QuantileDigestAggregationFunction.class.getClassLoader()); - List inputTypes = getInputTypes(valueType, arity); - QuantileDigestStateSerializer stateSerializer = new QuantileDigestStateSerializer(); - Type intermediateType = stateSerializer.getSerializedType(); - - AggregationMetadata metadata = new AggregationMetadata( - generateAggregationName(NAME, outputType.getTypeSignature(), inputTypes.stream().map(Type::getTypeSignature).collect(toImmutableList())), - createInputParameterMetadata(inputTypes), - getMethodHandle(valueType, arity), - COMBINE_FUNCTION, - OUTPUT_FUNCTION.bindTo(stateSerializer), - ImmutableList.of(new AccumulatorStateDescriptor( - QuantileDigestState.class, - stateSerializer, - new QuantileDigestStateFactory())), - outputType); - - GenericAccumulatorFactoryBinder factory = AccumulatorCompiler.generateAccumulatorFactoryBinder(metadata, classLoader); - return new InternalAggregationFunction(NAME, inputTypes, ImmutableList.of(intermediateType), outputType, true, true, factory); - } - - private static List getInputTypes(Type valueType, int arity) - { - switch (arity) { - case 1: - // weight and accuracy unspecified - return ImmutableList.of(valueType); - case 2: - // weight specified, accuracy unspecified - return ImmutableList.of(valueType, BIGINT); - case 3: - // weight and accuracy specified - return ImmutableList.of(valueType, BIGINT, DOUBLE); - default: - throw new IllegalArgumentException(format("Unsupported number of arguments: %s", arity)); - } - } - - private static MethodHandle getMethodHandle(Type valueType, int arity) - { - final MethodHandle inputFunction; - switch (valueType.getDisplayName()) { - case StandardTypes.DOUBLE: - inputFunction = INPUT_DOUBLE; - break; - case StandardTypes.REAL: - inputFunction = INPUT_REAL; - break; - case StandardTypes.BIGINT: - inputFunction = INPUT_BIGINT; - break; - default: - throw new IllegalArgumentException(format("Unsupported type %s supplied", valueType.getDisplayName())); - } - - switch (arity) { - case 1: - // weight and accuracy unspecified - return insertArguments(inputFunction, 2, DEFAULT_WEIGHT, DEFAULT_ACCURACY); - case 2: - // weight specified, accuracy unspecified - return insertArguments(inputFunction, 3, DEFAULT_ACCURACY); - case 3: - // weight and accuracy specified - return inputFunction; - default: - throw new IllegalArgumentException(format("Unsupported number of arguments: %s", arity)); - } + return "Returns a qdigest from the set of reals, bigints, doubles"; } - private static List createInputParameterMetadata(List valueTypes) + public static void inputDouble(StatisticalDigestState state, double value, long weight, double parameter) { - return ImmutableList.builder() - .add(new ParameterMetadata(STATE)) - .addAll(valueTypes.stream().map(valueType -> new ParameterMetadata(INPUT_CHANNEL, valueType)).collect(Collectors.toList())) - .build(); + inputBigint(state, doubleToSortableLong(value), weight, parameter); } - public static void inputDouble(QuantileDigestState state, double value, long weight, double accuracy) - { - inputBigint(state, doubleToSortableLong(value), weight, accuracy); - } - - public static void inputReal(QuantileDigestState state, long value, long weight, double accuracy) + public static void inputReal(StatisticalDigestState state, long value, long weight, double accuracy) { inputBigint(state, floatToSortableInt(intBitsToFloat((int) value)), weight, accuracy); } - public static void inputBigint(QuantileDigestState state, long value, long weight, double accuracy) + public static void inputBigint(StatisticalDigestState state, long value, long weight, double accuracy) { - QuantileDigest qdigest = getOrCreateQuantileDigest(state, verifyAccuracy(accuracy)); - state.addMemoryUsage(-qdigest.estimatedInMemorySizeInBytes()); - qdigest.add(value, verifyWeight(weight)); - state.addMemoryUsage(qdigest.estimatedInMemorySizeInBytes()); + StatisticalDigest digest = getOrCreateQuantileDigest(state, verifyAccuracy(accuracy)); + state.addMemoryUsage(-digest.estimatedInMemorySizeInBytes()); + digest.add(value, verifyWeight(weight)); + state.addMemoryUsage(digest.estimatedInMemorySizeInBytes()); } - private static QuantileDigest getOrCreateQuantileDigest(QuantileDigestState state, double accuracy) + private static StatisticalDigest getOrCreateQuantileDigest(StatisticalDigestState state, double parameter) { - QuantileDigest qdigest = state.getQuantileDigest(); - if (qdigest == null) { - qdigest = new QuantileDigest(accuracy); - state.setQuantileDigest(qdigest); - state.addMemoryUsage(qdigest.estimatedInMemorySizeInBytes()); + StatisticalDigest digest = state.getStatisticalDigest(); + if (digest == null) { + digest = createStatisticalQuantileDigest(new QuantileDigest(parameter)); + state.setStatisticalDigest(digest); + state.addMemoryUsage(digest.estimatedInMemorySizeInBytes()); } - return qdigest; - } - - public static void combineState(QuantileDigestState state, QuantileDigestState otherState) - { - QuantileDigest input = otherState.getQuantileDigest(); - - QuantileDigest previous = state.getQuantileDigest(); - if (previous == null) { - state.setQuantileDigest(input); - state.addMemoryUsage(input.estimatedInMemorySizeInBytes()); - } - else { - state.addMemoryUsage(-previous.estimatedInMemorySizeInBytes()); - previous.merge(input); - state.addMemoryUsage(previous.estimatedInMemorySizeInBytes()); - } - } - - public static void evaluateFinal(QuantileDigestStateSerializer serializer, QuantileDigestState state, BlockBuilder out) - { - serializer.serialize(state, out); + return digest; } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/StatisticalDigest.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/StatisticalDigest.java new file mode 100644 index 0000000000000..e359304443bf1 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/StatisticalDigest.java @@ -0,0 +1,42 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation; + +import com.facebook.presto.spi.PrestoException; +import io.airlift.slice.Slice; + +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; + +public interface StatisticalDigest +{ + default void add(double value, long weight) + { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "cannot add a double to a q-digest"); + } + + default void add(long value, long weight) + { + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "cannot add a long to a t-digest"); + } + + void merge(StatisticalDigest other); + + long estimatedInMemorySizeInBytes(); + + Slice serialize(); + + StatisticalDigest getDigest(); + + double getSize(); +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/StatisticalDigestAggregationFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/StatisticalDigestAggregationFunction.java new file mode 100644 index 0000000000000..04deee62f0fa9 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/StatisticalDigestAggregationFunction.java @@ -0,0 +1,267 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation; + +import com.facebook.presto.bytecode.DynamicClassLoader; +import com.facebook.presto.metadata.BoundVariables; +import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.metadata.SqlAggregationFunction; +import com.facebook.presto.operator.aggregation.state.StatisticalDigestState; +import com.facebook.presto.operator.aggregation.state.StatisticalDigestStateFactory; +import com.facebook.presto.operator.aggregation.state.StatisticalDigestStateSerializer; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.function.QualifiedFunctionName; +import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; +import com.facebook.presto.spi.type.StandardTypes; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeManager; +import com.facebook.presto.spi.type.TypeSignature; +import com.facebook.presto.spi.type.TypeSignatureParameter; +import com.google.common.collect.ImmutableList; + +import java.lang.invoke.MethodHandle; +import java.util.List; +import java.util.stream.Collectors; + +import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.operator.aggregation.AggregationMetadata.AccumulatorStateDescriptor; +import static com.facebook.presto.operator.aggregation.AggregationMetadata.ParameterMetadata; +import static com.facebook.presto.operator.aggregation.AggregationMetadata.ParameterMetadata.ParameterType.INPUT_CHANNEL; +import static com.facebook.presto.operator.aggregation.AggregationMetadata.ParameterMetadata.ParameterType.STATE; +import static com.facebook.presto.operator.aggregation.AggregationUtils.generateAggregationName; +import static com.facebook.presto.operator.scalar.QuantileDigestFunctions.DEFAULT_ACCURACY; +import static com.facebook.presto.operator.scalar.TDigestFunctions.DEFAULT_COMPRESSION; +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; +import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; +import static com.facebook.presto.spi.function.FunctionKind.AGGREGATE; +import static com.facebook.presto.spi.function.Signature.comparableTypeParameter; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.StandardTypes.QDIGEST; +import static com.facebook.presto.spi.type.StandardTypes.TDIGEST; +import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; +import static com.facebook.presto.util.Reflection.methodHandle; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.lang.String.format; +import static java.lang.invoke.MethodHandles.insertArguments; + +public abstract class StatisticalDigestAggregationFunction + extends SqlAggregationFunction +{ + private static final long DEFAULT_WEIGHT = 1L; + private final String name; + private final String type; + private final StatisticalDigestStateFactory factory; + + private static final MethodHandle T_DIGEST_INPUT_DOUBLE = methodHandle(TDigestAggregationFunction.class, + "inputDouble", + StatisticalDigestState.class, + double.class, + long.class, + double.class); + private static final MethodHandle QUANTILE_DIGEST_INPUT_DOUBLE = methodHandle( + QuantileDigestAggregationFunction.class, + "inputDouble", + StatisticalDigestState.class, + double.class, + long.class, + double.class); + private static final MethodHandle INPUT_REAL = methodHandle(QuantileDigestAggregationFunction.class, + "inputReal", + StatisticalDigestState.class, + long.class, + long.class, + double.class); + private static final MethodHandle INPUT_BIGINT = methodHandle( + QuantileDigestAggregationFunction.class, + "inputBigint", + StatisticalDigestState.class, + long.class, + long.class, + double.class); + + private static final MethodHandle COMBINE_FUNCTION = methodHandle( + StatisticalDigestAggregationFunction.class, + "combineState", + StatisticalDigestState.class, + StatisticalDigestState.class); + private static final MethodHandle OUTPUT_FUNCTION = methodHandle( + StatisticalDigestAggregationFunction.class, + "evaluateFinal", + StatisticalDigestStateSerializer.class, + StatisticalDigestState.class, + BlockBuilder.class); + + StatisticalDigestAggregationFunction(String name, String type, StatisticalDigestStateFactory factory, SqlFunctionVisibility visibility, TypeSignature... typeSignatures) + { + super(new Signature( + QualifiedFunctionName.of(DEFAULT_NAMESPACE, name), + AGGREGATE, + ImmutableList.of(comparableTypeParameter("V")), + ImmutableList.of(), + parseTypeSignature(type + "(V)"), + ImmutableList.copyOf(typeSignatures), + false), + visibility); + this.name = name; + this.type = type; + this.factory = factory; + } + + @Override + public InternalAggregationFunction specialize(BoundVariables boundVariables, int arity, TypeManager typeManager, FunctionManager functionManager) + { + Type valueType = boundVariables.getTypeVariable("V"); + Type outputType = typeManager.getParameterizedType( + type, + ImmutableList.of(TypeSignatureParameter.of(valueType.getTypeSignature()))); + return generateAggregation(name, valueType, outputType, arity); + } + + private InternalAggregationFunction generateAggregation(String name, Type valueType, Type outputType, int arity) + { + DynamicClassLoader classLoader = new DynamicClassLoader(StatisticalDigestAggregationFunction.class.getClassLoader()); + List inputTypes = getInputTypes(valueType, arity); + StatisticalDigestStateSerializer stateSerializer = new StatisticalDigestStateSerializer(); + Type intermediateType = stateSerializer.getSerializedType(); + + AggregationMetadata metadata = new AggregationMetadata( + generateAggregationName(name, outputType.getTypeSignature(), inputTypes.stream().map(Type::getTypeSignature).collect(toImmutableList())), + createInputParameterMetadata(inputTypes), + getInputMethodHandle(valueType, arity), + COMBINE_FUNCTION, + OUTPUT_FUNCTION.bindTo(stateSerializer), + ImmutableList.of(new AccumulatorStateDescriptor( + StatisticalDigestState.class, + stateSerializer, + factory)), + outputType); + + GenericAccumulatorFactoryBinder factory = AccumulatorCompiler.generateAccumulatorFactoryBinder(metadata, classLoader); + return new InternalAggregationFunction(name, inputTypes, ImmutableList.of(intermediateType), outputType, true, true, factory); + } + + private static List getInputTypes(Type valueType, int arity) + { + switch (arity) { + case 1: + // weight and accuracy unspecified + return ImmutableList.of(valueType); + case 2: + // weight specified, accuracy unspecified + return ImmutableList.of(valueType, BIGINT); + case 3: + // weight and accuracy specified + return ImmutableList.of(valueType, BIGINT, DOUBLE); + default: + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Unsupported number of arguments: %s", arity)); + } + } + + private MethodHandle getInputMethodHandle(Type valueType, int arity) + { + switch (type) { + case TDIGEST: + return getTDigestInputMethodHandle(valueType, arity); + case QDIGEST: + return getQuantileDigestInputMethodHandle(valueType, arity); + default: + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("%s must be a statistical digest", type)); + } + } + + private MethodHandle getTDigestInputMethodHandle(Type valueType, int arity) + { + final MethodHandle inputFunction; + switch (valueType.getDisplayName()) { + case StandardTypes.DOUBLE: + inputFunction = T_DIGEST_INPUT_DOUBLE; + break; + case StandardTypes.REAL: + throw new PrestoException(NOT_SUPPORTED, "Cannot operate on a t-digest with real numbers"); + case StandardTypes.BIGINT: + throw new PrestoException(NOT_SUPPORTED, "Cannot operate on a t-digest with longs"); + default: + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Unsupported type %s supplied", valueType.getDisplayName())); + } + return addArguments(inputFunction, arity, DEFAULT_COMPRESSION); + } + + private static MethodHandle getQuantileDigestInputMethodHandle(Type valueType, int arity) + { + final MethodHandle inputFunction; + switch (valueType.getDisplayName()) { + case StandardTypes.DOUBLE: + inputFunction = QUANTILE_DIGEST_INPUT_DOUBLE; + break; + case StandardTypes.REAL: + inputFunction = INPUT_REAL; + break; + case StandardTypes.BIGINT: + inputFunction = INPUT_BIGINT; + break; + default: + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Unsupported type %s supplied", valueType.getDisplayName())); + } + return addArguments(inputFunction, arity, DEFAULT_ACCURACY); + } + + private static MethodHandle addArguments(MethodHandle inputFunction, int arity, double parameter) + { + switch (arity) { + case 1: + // weight and accuracy unspecified + return insertArguments(inputFunction, 2, DEFAULT_WEIGHT, parameter); + case 2: + // weight specified, accuracy unspecified + return insertArguments(inputFunction, 3, parameter); + case 3: + // weight and accuracy specified + return inputFunction; + default: + throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Unsupported number of arguments: %s", arity)); + } + } + + private static List createInputParameterMetadata(List valueTypes) + { + return ImmutableList.builder() + .add(new ParameterMetadata(STATE)) + .addAll(valueTypes.stream().map(valueType -> new ParameterMetadata(INPUT_CHANNEL, valueType)).collect(Collectors.toList())) + .build(); + } + + public static void combineState(StatisticalDigestState state, StatisticalDigestState otherState) + { + StatisticalDigest input = otherState.getStatisticalDigest(); + StatisticalDigest previous = state.getStatisticalDigest(); + + if (previous == null) { + state.setStatisticalDigest(input); + state.addMemoryUsage(input.estimatedInMemorySizeInBytes()); + } + else { + state.addMemoryUsage(-previous.estimatedInMemorySizeInBytes()); + previous.merge(input); + state.addMemoryUsage(previous.estimatedInMemorySizeInBytes()); + } + } + + public static void evaluateFinal(StatisticalDigestStateSerializer serializer, StatisticalDigestState state, BlockBuilder out) + { + serializer.serialize(state, out); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/StatisticalDigestFactory.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/StatisticalDigestFactory.java new file mode 100644 index 0000000000000..ea07dce63c61b --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/StatisticalDigestFactory.java @@ -0,0 +1,135 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation; + +import com.facebook.airlift.stats.QuantileDigest; +import com.facebook.presto.tdigest.TDigest; +import io.airlift.slice.Slice; + +import static com.google.common.base.Preconditions.checkArgument; + +public class StatisticalDigestFactory +{ + private StatisticalDigestFactory() + { + } + + public static StatisticalDigest createStatisticalTDigest(TDigest tDigest) + { + return new StatisticalTDigest(tDigest); + } + + public static StatisticalDigest createStatisticalQuantileDigest(QuantileDigest quantileDigest) + { + return new StatisticalQuantileDigest(quantileDigest); + } + + public static class StatisticalTDigest + implements StatisticalDigest + { + private final TDigest tDigest; + + public StatisticalTDigest(TDigest tDigest) + { + this.tDigest = tDigest; + } + + @Override + public void add(double value, long weight) + { + tDigest.add(value, weight); + } + + @Override + public void merge(StatisticalDigest other) + { + checkArgument(other instanceof StatisticalTDigest); + StatisticalTDigest toMerge = (StatisticalTDigest) other; + tDigest.merge(toMerge.tDigest); + } + + @Override + public double getSize() + { + return tDigest.getSize(); + } + + @Override + public long estimatedInMemorySizeInBytes() + { + return tDigest.estimatedInMemorySizeInBytes(); + } + + @Override + public Slice serialize() + { + return tDigest.serialize(); + } + + @Override + public StatisticalDigest getDigest() + { + return this; + } + } + + public static class StatisticalQuantileDigest + implements StatisticalDigest + { + private final QuantileDigest qdigest; + + public StatisticalQuantileDigest(QuantileDigest qdigest) + { + this.qdigest = qdigest; + } + + @Override + public void add(long value, long weight) + { + qdigest.add(value, weight); + } + + @Override + public void merge(StatisticalDigest other) + { + checkArgument(other instanceof StatisticalQuantileDigest); + StatisticalQuantileDigest toMerge = (StatisticalQuantileDigest) other; + this.qdigest.merge(toMerge.qdigest); + } + + @Override + public double getSize() + { + return qdigest.getCount(); + } + + @Override + public long estimatedInMemorySizeInBytes() + { + return qdigest.estimatedInMemorySizeInBytes(); + } + + @Override + public Slice serialize() + { + return qdigest.serialize(); + } + + @Override + public StatisticalDigest getDigest() + { + return this; + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/SumDataSizeForStats.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/SumDataSizeForStats.java index b2682ba6de143..5d82d3964356a 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/SumDataSizeForStats.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/SumDataSizeForStats.java @@ -28,7 +28,9 @@ import com.facebook.presto.spi.type.BigintType; import com.facebook.presto.spi.type.StandardTypes; -@AggregationFunction(value = SumDataSizeForStats.NAME, hidden = true) +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; + +@AggregationFunction(value = SumDataSizeForStats.NAME, visibility = HIDDEN) public final class SumDataSizeForStats { public static final String NAME = "$internal$sum_data_size_for_stats"; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/TDigestAggregationFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/TDigestAggregationFunction.java new file mode 100644 index 0000000000000..ce32f3cc51d5f --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/TDigestAggregationFunction.java @@ -0,0 +1,73 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation; + +import com.facebook.presto.operator.aggregation.state.StatisticalDigestState; +import com.facebook.presto.spi.type.StandardTypes; +import com.facebook.presto.spi.type.TypeSignature; + +import static com.facebook.presto.operator.aggregation.StatisticalDigestFactory.createStatisticalTDigest; +import static com.facebook.presto.operator.aggregation.state.StatisticalDigestStateFactory.createTDigestFactory; +import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.EXPERIMENTAL; +import static com.facebook.presto.spi.type.StandardTypes.TDIGEST; +import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; +import static com.facebook.presto.tdigest.TDigest.createTDigest; +import static com.facebook.presto.util.Failures.checkCondition; + +public class TDigestAggregationFunction + extends StatisticalDigestAggregationFunction +{ + public static final TDigestAggregationFunction TDIGEST_AGG = new TDigestAggregationFunction(parseTypeSignature("V")); + public static final TDigestAggregationFunction TDIGEST_AGG_WITH_WEIGHT = new TDigestAggregationFunction(parseTypeSignature("V"), parseTypeSignature(StandardTypes.BIGINT)); + public static final TDigestAggregationFunction TDIGEST_AGG_WITH_WEIGHT_AND_COMPRESSION = new TDigestAggregationFunction( + parseTypeSignature("V"), + parseTypeSignature(StandardTypes.BIGINT), + parseTypeSignature(StandardTypes.DOUBLE)); + + public static final String NAME = "tdigest_agg"; + + private TDigestAggregationFunction(TypeSignature... typeSignatures) + { + super(NAME, TDIGEST, createTDigestFactory(), EXPERIMENTAL, typeSignatures); + } + + @Override + public String getDescription() + { + return "Returns a tdigest from the set of doubles"; + } + + public static void inputDouble(StatisticalDigestState state, double value, long weight, double compression) + { + checkCondition(weight > 0, INVALID_FUNCTION_ARGUMENT, "Weight must be > 0, was %s", weight); + checkCondition(compression > 0, INVALID_FUNCTION_ARGUMENT, "Compression factor must be positive, was %s", compression); + + StatisticalDigest digest = getOrCreateTDigest(state, compression); + state.addMemoryUsage(-digest.estimatedInMemorySizeInBytes()); + digest.add(value, weight); + state.addMemoryUsage(digest.estimatedInMemorySizeInBytes()); + } + + private static StatisticalDigest getOrCreateTDigest(StatisticalDigestState state, double compression) + { + StatisticalDigest digest = state.getStatisticalDigest(); + if (digest == null) { + digest = createStatisticalTDigest(createTDigest(compression)); + state.setStatisticalDigest(digest); + state.addMemoryUsage(digest.estimatedInMemorySizeInBytes()); + } + return digest; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateFactory.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateFactory.java deleted file mode 100644 index 688a4b681e3c4..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateFactory.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.operator.aggregation.state; - -import com.facebook.airlift.stats.QuantileDigest; -import com.facebook.presto.array.ObjectBigArray; -import com.facebook.presto.spi.function.AccumulatorStateFactory; -import org.openjdk.jol.info.ClassLayout; - -import static java.util.Objects.requireNonNull; - -public class QuantileDigestStateFactory - implements AccumulatorStateFactory -{ - @Override - public QuantileDigestState createSingleState() - { - return new SingleQuantileDigestState(); - } - - @Override - public Class getSingleStateClass() - { - return SingleQuantileDigestState.class; - } - - @Override - public QuantileDigestState createGroupedState() - { - return new GroupedQuantileDigestState(); - } - - @Override - public Class getGroupedStateClass() - { - return GroupedQuantileDigestState.class; - } - - public static class GroupedQuantileDigestState - extends AbstractGroupedAccumulatorState - implements QuantileDigestState - { - private static final int INSTANCE_SIZE = ClassLayout.parseClass(GroupedQuantileDigestState.class).instanceSize(); - private final ObjectBigArray qdigests = new ObjectBigArray<>(); - private long size; - - @Override - public void ensureCapacity(long size) - { - qdigests.ensureCapacity(size); - } - - @Override - public QuantileDigest getQuantileDigest() - { - return qdigests.get(getGroupId()); - } - - @Override - public void setQuantileDigest(QuantileDigest value) - { - requireNonNull(value, "value is null"); - qdigests.set(getGroupId(), value); - } - - @Override - public void addMemoryUsage(int value) - { - size += value; - } - - @Override - public long getEstimatedSize() - { - return INSTANCE_SIZE + size + qdigests.sizeOf(); - } - } - - public static class SingleQuantileDigestState - implements QuantileDigestState - { - private static final int INSTANCE_SIZE = ClassLayout.parseClass(SingleQuantileDigestState.class).instanceSize(); - private QuantileDigest qdigest; - - @Override - public QuantileDigest getQuantileDigest() - { - return qdigest; - } - - @Override - public void setQuantileDigest(QuantileDigest value) - { - qdigest = value; - } - - @Override - public void addMemoryUsage(int value) - { - // noop - } - - @Override - public long getEstimatedSize() - { - long estimatedSize = INSTANCE_SIZE; - if (qdigest != null) { - estimatedSize += qdigest.estimatedInMemorySizeInBytes(); - } - return estimatedSize; - } - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestState.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/StatisticalDigestState.java similarity index 67% rename from presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestState.java rename to presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/StatisticalDigestState.java index 9b4a080f0e4b8..dd26e39ba21b6 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestState.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/StatisticalDigestState.java @@ -13,15 +13,18 @@ */ package com.facebook.presto.operator.aggregation.state; -import com.facebook.airlift.stats.QuantileDigest; +import com.facebook.presto.operator.aggregation.StatisticalDigest; import com.facebook.presto.spi.function.AccumulatorState; +import io.airlift.slice.Slice; -public interface QuantileDigestState +public interface StatisticalDigestState extends AccumulatorState { - QuantileDigest getQuantileDigest(); + StatisticalDigest getStatisticalDigest(); - void setQuantileDigest(QuantileDigest value); + void setStatisticalDigest(StatisticalDigest value); - void addMemoryUsage(int value); + void addMemoryUsage(long value); + + StatisticalDigest deserialize(Slice slice); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/StatisticalDigestStateFactory.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/StatisticalDigestStateFactory.java new file mode 100644 index 0000000000000..53b03b62c629b --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/StatisticalDigestStateFactory.java @@ -0,0 +1,174 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.operator.aggregation.state; + +import com.facebook.airlift.stats.QuantileDigest; +import com.facebook.presto.array.ObjectBigArray; +import com.facebook.presto.operator.aggregation.StatisticalDigest; +import com.facebook.presto.spi.function.AccumulatorStateFactory; +import com.facebook.presto.tdigest.TDigest; +import io.airlift.slice.Slice; +import org.openjdk.jol.info.ClassLayout; + +import java.util.function.Function; + +import static com.facebook.presto.operator.aggregation.StatisticalDigestFactory.createStatisticalQuantileDigest; +import static com.facebook.presto.operator.aggregation.StatisticalDigestFactory.createStatisticalTDigest; +import static com.facebook.presto.tdigest.TDigest.createTDigest; +import static java.util.Objects.requireNonNull; + +public class StatisticalDigestStateFactory + implements AccumulatorStateFactory +{ + private final Function> deserializer; + + public static StatisticalDigestStateFactory createTDigestFactory() + { + return new StatisticalDigestStateFactory((slice) -> createStatisticalTDigest(createTDigest(slice))); + } + + public static StatisticalDigestStateFactory createQuantileDigestFactory() + { + return new StatisticalDigestStateFactory((slice) -> createStatisticalQuantileDigest(new QuantileDigest(slice))); + } + + private StatisticalDigestStateFactory(Function> deserializer) + { + this.deserializer = deserializer; + } + + @Override + public StatisticalDigestState createSingleState() + { + return new SingleStatisticalDigestState(deserializer); + } + + @Override + public Class getSingleStateClass() + { + return SingleStatisticalDigestState.class; + } + + @Override + public StatisticalDigestState createGroupedState() + { + return new GroupedStatisticalDigestState(deserializer); + } + + @Override + public Class getGroupedStateClass() + { + return GroupedStatisticalDigestState.class; + } + + public static class GroupedStatisticalDigestState + extends AbstractGroupedAccumulatorState + implements StatisticalDigestState + { + private static final int INSTANCE_SIZE = ClassLayout.parseClass(GroupedStatisticalDigestState.class).instanceSize(); + private final ObjectBigArray> digests = new ObjectBigArray<>(); + private long size; + private final Function> deserializer; + + public GroupedStatisticalDigestState(Function> deserializer) + { + this.deserializer = deserializer; + } + + @Override + public void ensureCapacity(long size) + { + digests.ensureCapacity(size); + } + + @Override + public StatisticalDigest getStatisticalDigest() + { + return digests.get(getGroupId()); + } + + @Override + public void setStatisticalDigest(StatisticalDigest value) + { + requireNonNull(value, "value is null"); + digests.set(getGroupId(), value); + } + + @Override + public void addMemoryUsage(long value) + { + size += value; + } + + @Override + public StatisticalDigest deserialize(Slice slice) + { + return deserializer.apply(slice); + } + + @Override + public long getEstimatedSize() + { + return INSTANCE_SIZE + size + digests.sizeOf(); + } + } + + public static class SingleStatisticalDigestState + implements StatisticalDigestState + { + private static final int INSTANCE_SIZE = ClassLayout.parseClass(SingleStatisticalDigestState.class).instanceSize(); + private final Function> deserializer; + private StatisticalDigest digest; + + public SingleStatisticalDigestState(Function> deserializer) + { + this.deserializer = deserializer; + } + + @Override + public StatisticalDigest getStatisticalDigest() + { + return digest; + } + + @Override + public void setStatisticalDigest(StatisticalDigest value) + { + digest = value; + } + + @Override + public void addMemoryUsage(long value) + { + // noop + } + + @Override + public StatisticalDigest deserialize(Slice slice) + { + return deserializer.apply(slice); + } + + @Override + public long getEstimatedSize() + { + long estimatedSize = INSTANCE_SIZE; + if (digest != null) { + estimatedSize += digest.estimatedInMemorySizeInBytes(); + } + return estimatedSize; + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateSerializer.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/StatisticalDigestStateSerializer.java similarity index 67% rename from presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateSerializer.java rename to presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/StatisticalDigestStateSerializer.java index 05c2231e0672f..cb263bdfa0c97 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/QuantileDigestStateSerializer.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/state/StatisticalDigestStateSerializer.java @@ -11,9 +11,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.facebook.presto.operator.aggregation.state; -import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.function.AccumulatorStateSerializer; @@ -21,8 +21,8 @@ import static com.facebook.presto.spi.type.VarbinaryType.VARBINARY; -public class QuantileDigestStateSerializer - implements AccumulatorStateSerializer +public class StatisticalDigestStateSerializer + implements AccumulatorStateSerializer { @Override public Type getSerializedType() @@ -31,19 +31,19 @@ public Type getSerializedType() } @Override - public void serialize(QuantileDigestState state, BlockBuilder out) + public void serialize(StatisticalDigestState state, BlockBuilder out) { - if (state.getQuantileDigest() == null) { + if (state.getStatisticalDigest() == null) { out.appendNull(); } else { - VARBINARY.writeSlice(out, state.getQuantileDigest().serialize()); + VARBINARY.writeSlice(out, state.getStatisticalDigest().serialize()); } } @Override - public void deserialize(Block block, int index, QuantileDigestState state) + public void deserialize(Block block, int index, StatisticalDigestState state) { - state.setQuantileDigest(new QuantileDigest(VARBINARY.getSlice(block, index))); + state.setStatisticalDigest(state.deserialize(VARBINARY.getSlice(block, index))); } } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/AbstractGreatestLeast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/AbstractGreatestLeast.java index 0ac39aafe9740..180f466b22aee 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/AbstractGreatestLeast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/AbstractGreatestLeast.java @@ -30,6 +30,7 @@ import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -87,9 +88,9 @@ protected AbstractGreatestLeast(QualifiedFunctionName name, OperatorType operato } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return SqlFunctionVisibility.PUBLIC; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ApplyFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ApplyFunction.java index c681f9c68f83e..8904fd554ceeb 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ApplyFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ApplyFunction.java @@ -19,6 +19,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.gen.lambda.UnaryFunctionInterface; @@ -31,6 +32,7 @@ import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.util.Reflection.methodHandle; import static com.google.common.primitives.Primitives.wrap; @@ -58,9 +60,9 @@ private ApplyFunction() } @Override - public boolean isHidden() + public final SqlFunctionVisibility getVisibility() { - return true; + return HIDDEN; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConcatFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConcatFunction.java index 9179b02329a7e..2869fd3c94158 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConcatFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConcatFunction.java @@ -24,6 +24,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.gen.VarArgsToArrayAdapterGenerator; @@ -37,6 +38,7 @@ import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.sql.gen.VarArgsToArrayAdapterGenerator.generateVarArgsToArrayAdapter; import static com.facebook.presto.util.Reflection.methodHandle; @@ -66,9 +68,9 @@ private ArrayConcatFunction() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return PUBLIC; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConstructor.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConstructor.java index 99c449b84b2cb..7f24d60c52352 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConstructor.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConstructor.java @@ -31,6 +31,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.gen.CallSiteBinder; @@ -58,6 +59,7 @@ import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.sql.gen.SqlTypeBytecodeExpression.constantType; import static com.facebook.presto.util.CompilerUtils.defineClass; @@ -85,9 +87,9 @@ public ArrayConstructor() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return true; + return HIDDEN; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayFlattenFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayFlattenFunction.java index cab59b576e049..d0a6dd3b3b528 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayFlattenFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayFlattenFunction.java @@ -21,6 +21,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -33,6 +34,7 @@ import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.util.Reflection.methodHandle; import static java.lang.Math.toIntExact; @@ -56,9 +58,9 @@ private ArrayFlattenFunction() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return PUBLIC; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayFunctions.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayFunctions.java index e11b6da456254..d9eed27f4d200 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayFunctions.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayFunctions.java @@ -19,6 +19,7 @@ import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.type.ArrayType; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; import static com.facebook.presto.type.UnknownType.UNKNOWN; public final class ArrayFunctions @@ -27,7 +28,7 @@ private ArrayFunctions() { } - @ScalarFunction(hidden = true) + @ScalarFunction(visibility = HIDDEN) @SqlType("array(unknown)") public static Block arrayConstructor() { diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayJoin.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayJoin.java index 24ed2b6754f53..a4d21c1f516e7 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayJoin.java @@ -28,6 +28,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -52,6 +53,7 @@ import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.spi.type.VarcharType.VARCHAR; import static com.facebook.presto.util.Reflection.methodHandle; @@ -129,9 +131,9 @@ public ArrayJoinWithNullReplacement() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return PUBLIC; } @Override @@ -177,9 +179,9 @@ public static Object createState() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return PUBLIC; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayReduceFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayReduceFunction.java index d6fec733041bc..e820de95a8cd6 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayReduceFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayReduceFunction.java @@ -20,6 +20,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.gen.lambda.BinaryFunctionInterface; @@ -35,6 +36,7 @@ import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.spi.type.TypeUtils.readNativeValue; import static com.facebook.presto.util.Reflection.methodHandle; @@ -59,9 +61,9 @@ private ArrayReduceFunction() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return PUBLIC; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToElementConcatFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToElementConcatFunction.java index 6f7526cb9ff62..1774939234698 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToElementConcatFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToElementConcatFunction.java @@ -20,6 +20,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableList; @@ -31,6 +32,7 @@ import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.util.Reflection.methodHandle; @@ -58,9 +60,9 @@ public ArrayToElementConcatFunction() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return PUBLIC; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayTransformFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayTransformFunction.java index 9bcb66d462191..5e295ef2d03ef 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayTransformFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayTransformFunction.java @@ -31,6 +31,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -86,9 +87,9 @@ private ArrayTransformFunction() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return SqlFunctionVisibility.PUBLIC; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/CombineHashFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/CombineHashFunction.java index a5f3d3bd57460..5aabde40e990a 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/CombineHashFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/CombineHashFunction.java @@ -17,11 +17,13 @@ import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.type.StandardTypes; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; + public final class CombineHashFunction { private CombineHashFunction() {} - @ScalarFunction(value = "combine_hash", hidden = true) + @ScalarFunction(value = "combine_hash", visibility = HIDDEN) @SqlType(StandardTypes.BIGINT) public static long getHash(@SqlType(StandardTypes.BIGINT) long previousHashValue, @SqlType(StandardTypes.BIGINT) long value) { diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ConcatFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ConcatFunction.java index 8409aa393a032..292be0756feb5 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ConcatFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ConcatFunction.java @@ -29,6 +29,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.spi.type.TypeSignature; import com.google.common.collect.ImmutableList; @@ -89,9 +90,9 @@ private ConcatFunction(TypeSignature type, String description) } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return SqlFunctionVisibility.PUBLIC; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/DateTimeFunctions.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/DateTimeFunctions.java index d0352652e64d1..a9f66e0895e10 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/DateTimeFunctions.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/DateTimeFunctions.java @@ -40,6 +40,7 @@ import static com.facebook.presto.operator.scalar.QuarterOfYearDateTimeField.QUARTER_OF_YEAR; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; import static com.facebook.presto.spi.type.DateTimeEncoding.packDateTimeWithZone; import static com.facebook.presto.spi.type.DateTimeEncoding.unpackMillisUtc; import static com.facebook.presto.spi.type.DateTimeEncoding.unpackZoneKey; @@ -266,7 +267,7 @@ public static long fromISO8601Date(ConnectorSession session, @SqlType("varchar(x return MILLISECONDS.toDays(dateTime.getMillis()); } - @ScalarFunction(value = "at_timezone", hidden = true) + @ScalarFunction(value = "at_timezone", visibility = HIDDEN) @LiteralParameters("x") @SqlType(StandardTypes.TIME_WITH_TIME_ZONE) public static long timeAtTimeZone(ConnectorSession session, @SqlType(StandardTypes.TIME_WITH_TIME_ZONE) long timeWithTimeZone, @SqlType("varchar(x)") Slice zoneId) @@ -274,7 +275,7 @@ public static long timeAtTimeZone(ConnectorSession session, @SqlType(StandardTyp return timeAtTimeZone(session, timeWithTimeZone, getTimeZoneKey(zoneId.toStringUtf8())); } - @ScalarFunction(value = "at_timezone", hidden = true) + @ScalarFunction(value = "at_timezone", visibility = HIDDEN) @SqlType(StandardTypes.TIME_WITH_TIME_ZONE) public static long timeAtTimeZone(ConnectorSession session, @SqlType(StandardTypes.TIME_WITH_TIME_ZONE) long timeWithTimeZone, @SqlType(StandardTypes.INTERVAL_DAY_TO_SECOND) long zoneOffset) { @@ -283,7 +284,7 @@ public static long timeAtTimeZone(ConnectorSession session, @SqlType(StandardTyp return timeAtTimeZone(session, timeWithTimeZone, getTimeZoneKeyForOffset(zoneOffsetMinutes)); } - @ScalarFunction(value = "at_timezone", hidden = true) + @ScalarFunction(value = "at_timezone", visibility = HIDDEN) @LiteralParameters("x") @SqlType(StandardTypes.TIMESTAMP_WITH_TIME_ZONE) public static long timestampAtTimeZone(@SqlType(StandardTypes.TIMESTAMP_WITH_TIME_ZONE) long timestampWithTimeZone, @SqlType("varchar(x)") Slice zoneId) @@ -291,7 +292,7 @@ public static long timestampAtTimeZone(@SqlType(StandardTypes.TIMESTAMP_WITH_TIM return packDateTimeWithZone(unpackMillisUtc(timestampWithTimeZone), zoneId.toStringUtf8()); } - @ScalarFunction(value = "at_timezone", hidden = true) + @ScalarFunction(value = "at_timezone", visibility = HIDDEN) @SqlType(StandardTypes.TIMESTAMP_WITH_TIME_ZONE) public static long timestampAtTimeZone(@SqlType(StandardTypes.TIMESTAMP_WITH_TIME_ZONE) long timestampWithTimeZone, @SqlType(StandardTypes.INTERVAL_DAY_TO_SECOND) long zoneOffset) { diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ElementToArrayConcatFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ElementToArrayConcatFunction.java index 63a2ca140253b..6521904e47ea0 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ElementToArrayConcatFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ElementToArrayConcatFunction.java @@ -20,6 +20,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableList; @@ -31,6 +32,7 @@ import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.util.Reflection.methodHandle; @@ -58,9 +60,9 @@ public ElementToArrayConcatFunction() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return PUBLIC; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/FailureFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/FailureFunction.java index ba2980717febe..42b459f535b18 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/FailureFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/FailureFunction.java @@ -23,6 +23,8 @@ import com.facebook.presto.spi.type.StandardTypes; import io.airlift.slice.Slice; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; + public final class FailureFunction { private static final JsonCodec JSON_CODEC = JsonCodec.jsonCodec(FailureInfo.class); @@ -31,7 +33,7 @@ private FailureFunction() {} // We shouldn't be using UNKNOWN as an explicit type. This will be fixed when we fix type inference @Description("Decodes json to an exception and throws it") - @ScalarFunction(value = "fail", hidden = true) + @ScalarFunction(value = "fail", visibility = HIDDEN) @SqlType("unknown") public static boolean failWithException(@SqlType(StandardTypes.JSON) Slice failureInfoSlice) { @@ -42,7 +44,7 @@ public static boolean failWithException(@SqlType(StandardTypes.JSON) Slice failu // This function is only used to propagate optimization failures. @Description("Decodes json to an exception and throws it with supplied errorCode") - @ScalarFunction(value = "fail", hidden = true) + @ScalarFunction(value = "fail", visibility = HIDDEN) @SqlType("unknown") public static boolean failWithException( @SqlType(StandardTypes.INTEGER) long errorCode, @@ -59,7 +61,7 @@ public static boolean failWithException( } @Description("Throws an exception with a given message") - @ScalarFunction(value = "fail", hidden = true) + @ScalarFunction(value = "fail", visibility = HIDDEN) @SqlType("unknown") public static boolean fail(@SqlType(StandardTypes.VARCHAR) Slice message) { @@ -67,7 +69,7 @@ public static boolean fail(@SqlType(StandardTypes.VARCHAR) Slice message) } @Description("Throws an exception with a given error code and message") - @ScalarFunction(value = "fail", hidden = true) + @ScalarFunction(value = "fail", visibility = HIDDEN) @SqlType("unknown") public static boolean fail( @SqlType(StandardTypes.INTEGER) long errorCode, diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/InvokeFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/InvokeFunction.java index 0eb8f3817720f..11defe51dd28e 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/InvokeFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/InvokeFunction.java @@ -20,6 +20,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.facebook.presto.sql.gen.lambda.LambdaFunctionInterface; @@ -30,6 +31,7 @@ import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.functionTypeArgumentProperty; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.util.Reflection.methodHandle; import static com.google.common.primitives.Primitives.wrap; @@ -57,9 +59,9 @@ private InvokeFunction() } @Override - public boolean isHidden() + public final SqlFunctionVisibility getVisibility() { - return true; + return HIDDEN; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToArrayCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToArrayCast.java index 504f10c2c7443..51e097c7eb8ff 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToArrayCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToArrayCast.java @@ -18,6 +18,7 @@ import com.facebook.presto.metadata.SqlScalarFunction; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableList; @@ -26,6 +27,7 @@ import static com.facebook.presto.operator.scalar.JsonToArrayCast.JSON_TO_ARRAY; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; public final class JsonStringToArrayCast @@ -60,9 +62,9 @@ public boolean isDeterministic() } @Override - public final boolean isHidden() + public final SqlFunctionVisibility getVisibility() { - return true; + return HIDDEN; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToMapCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToMapCast.java index bf1e710f868ec..b541966322924 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToMapCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToMapCast.java @@ -18,6 +18,7 @@ import com.facebook.presto.metadata.SqlScalarFunction; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableList; @@ -27,6 +28,7 @@ import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.spi.function.Signature.comparableTypeParameter; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; public final class JsonStringToMapCast @@ -61,9 +63,9 @@ public boolean isDeterministic() } @Override - public final boolean isHidden() + public final SqlFunctionVisibility getVisibility() { - return true; + return HIDDEN; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToRowCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToRowCast.java index e958f5003c27e..bffe82659f3db 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToRowCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToRowCast.java @@ -18,6 +18,7 @@ import com.facebook.presto.metadata.SqlScalarFunction; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableList; @@ -26,6 +27,7 @@ import static com.facebook.presto.operator.scalar.JsonToRowCast.JSON_TO_ROW; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.spi.function.Signature.withVariadicBound; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; public final class JsonStringToRowCast @@ -60,9 +62,9 @@ public boolean isDeterministic() } @Override - public final boolean isHidden() + public final SqlFunctionVisibility getVisibility() { - return true; + return HIDDEN; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConcatFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConcatFunction.java index 6a911588d1834..0da833fb20682 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConcatFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConcatFunction.java @@ -25,6 +25,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; @@ -41,6 +42,7 @@ import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.sql.gen.VarArgsToArrayAdapterGenerator.generateVarArgsToArrayAdapter; import static com.facebook.presto.util.Reflection.methodHandle; @@ -71,9 +73,9 @@ private MapConcatFunction() } @Override - public boolean isHidden() + public final SqlFunctionVisibility getVisibility() { - return false; + return PUBLIC; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConstructor.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConstructor.java index f390481fc954a..5e8bf897a9e85 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConstructor.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConstructor.java @@ -28,6 +28,7 @@ import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -45,6 +46,7 @@ import static com.facebook.presto.spi.function.OperatorType.INDETERMINATE; import static com.facebook.presto.spi.function.Signature.comparableTypeParameter; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static com.facebook.presto.spi.type.StandardTypes.MAP; import static com.facebook.presto.spi.type.TypeUtils.readNativeValue; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypeSignatures; @@ -86,9 +88,9 @@ public MapConstructor() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return PUBLIC; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapElementAtFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapElementAtFunction.java index acf7438ed63e5..2785f59bb7eeb 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapElementAtFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapElementAtFunction.java @@ -23,6 +23,7 @@ import com.facebook.presto.spi.function.OperatorType; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableList; @@ -35,6 +36,7 @@ import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.spi.type.TypeUtils.readNativeValue; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; @@ -64,9 +66,9 @@ protected MapElementAtFunction() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return PUBLIC; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapFilterFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapFilterFunction.java index 981a1c25334a5..fd021f0c41efb 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapFilterFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapFilterFunction.java @@ -32,6 +32,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; @@ -92,9 +93,9 @@ private MapFilterFunction() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return SqlFunctionVisibility.PUBLIC; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformKeyFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformKeyFunction.java index 9706ccb6d4c32..b91145b663f1b 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformKeyFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformKeyFunction.java @@ -36,6 +36,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; @@ -102,9 +103,9 @@ private MapTransformKeyFunction() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return SqlFunctionVisibility.PUBLIC; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformValueFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformValueFunction.java index ac3c8fcd39407..72fb4762c32b0 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformValueFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformValueFunction.java @@ -35,6 +35,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; @@ -100,9 +101,9 @@ private MapTransformValueFunction() } @Override - public boolean isHidden() + public final SqlFunctionVisibility getVisibility() { - return false; + return SqlFunctionVisibility.PUBLIC; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapZipWithFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapZipWithFunction.java index 73299327da607..a4569156f0de3 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapZipWithFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapZipWithFunction.java @@ -23,6 +23,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.MapType; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; @@ -39,6 +40,7 @@ import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.spi.type.TypeUtils.readNativeValue; import static com.facebook.presto.spi.type.TypeUtils.writeNativeValue; @@ -66,9 +68,9 @@ private MapZipWithFunction() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return PUBLIC; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ParametricScalar.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ParametricScalar.java index 107d5739b1d71..7a4db8a575ba2 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ParametricScalar.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ParametricScalar.java @@ -20,6 +20,7 @@ import com.facebook.presto.operator.scalar.annotations.ParametricScalarImplementation; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.TypeManager; import com.google.common.annotations.VisibleForTesting; @@ -50,9 +51,9 @@ public ParametricScalar( } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return details.isHidden(); + return details.getVisibility(); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ScalarHeader.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ScalarHeader.java index c415090493a73..b6b5e33fa6c00 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ScalarHeader.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ScalarHeader.java @@ -13,19 +13,21 @@ */ package com.facebook.presto.operator.scalar; +import com.facebook.presto.spi.function.SqlFunctionVisibility; + import java.util.Optional; public class ScalarHeader { private final Optional description; - private final boolean hidden; + private final SqlFunctionVisibility visibility; private final boolean deterministic; private final boolean calledOnNullInput; - public ScalarHeader(Optional description, boolean hidden, boolean deterministic, boolean calledOnNullInput) + public ScalarHeader(Optional description, SqlFunctionVisibility visibility, boolean deterministic, boolean calledOnNullInput) { this.description = description; - this.hidden = hidden; + this.visibility = visibility; this.deterministic = deterministic; this.calledOnNullInput = calledOnNullInput; } @@ -35,9 +37,9 @@ public Optional getDescription() return description; } - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return hidden; + return visibility; } public boolean isDeterministic() diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/SessionFunctions.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/SessionFunctions.java index 560f994f33597..1e76777731db4 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/SessionFunctions.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/SessionFunctions.java @@ -20,13 +20,14 @@ import com.facebook.presto.spi.type.StandardTypes; import io.airlift.slice.Slice; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; import static io.airlift.slice.Slices.utf8Slice; public final class SessionFunctions { private SessionFunctions() {} - @ScalarFunction(value = "$current_user", hidden = true) + @ScalarFunction(value = "$current_user", visibility = HIDDEN) @Description("current user") @SqlType(StandardTypes.VARCHAR) public static Slice currentUser(ConnectorSession session) diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/TDigestFunctions.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/TDigestFunctions.java new file mode 100644 index 0000000000000..2b816b6be6c7f --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/TDigestFunctions.java @@ -0,0 +1,76 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.scalar; + +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.function.Description; +import com.facebook.presto.spi.function.ScalarFunction; +import com.facebook.presto.spi.function.SqlType; +import com.facebook.presto.spi.type.StandardTypes; +import com.facebook.presto.tdigest.TDigest; +import io.airlift.slice.Slice; + +import static com.facebook.presto.spi.function.SqlFunctionVisibility.EXPERIMENTAL; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.tdigest.TDigest.createTDigest; + +public final class TDigestFunctions +{ + public static final double DEFAULT_COMPRESSION = 100; + + private TDigestFunctions() {} + + @ScalarFunction(value = "value_at_quantile", visibility = EXPERIMENTAL) + @Description("Given an input q between [0, 1], find the value whose rank in the sorted sequence of the n values represented by the tdigest is qn.") + @SqlType(StandardTypes.DOUBLE) + public static double valueAtQuantileDouble(@SqlType("tdigest(double)") Slice input, @SqlType(StandardTypes.DOUBLE) double quantile) + { + return createTDigest(input).getQuantile(quantile); + } + + @ScalarFunction(value = "values_at_quantiles", visibility = EXPERIMENTAL) + @Description("For each input q between [0, 1], find the value whose rank in the sorted sequence of the n values represented by the tdigest is qn.") + @SqlType("array(double)") + public static Block valuesAtQuantilesDouble(@SqlType("tdigest(double)") Slice input, @SqlType("array(double)") Block percentilesArrayBlock) + { + TDigest tDigest = createTDigest(input); + BlockBuilder output = DOUBLE.createBlockBuilder(null, percentilesArrayBlock.getPositionCount()); + for (int i = 0; i < percentilesArrayBlock.getPositionCount(); i++) { + DOUBLE.writeDouble(output, tDigest.getQuantile(DOUBLE.getDouble(percentilesArrayBlock, i))); + } + return output.build(); + } + + @ScalarFunction(value = "quantile_at_value", visibility = EXPERIMENTAL) + @Description("Given an input x between min/max values of t-digest, find which quantile is represented by that value") + @SqlType(StandardTypes.DOUBLE) + public static double quantileAtValueDouble(@SqlType("tdigest(double)") Slice input, @SqlType(StandardTypes.DOUBLE) double value) + { + return createTDigest(input).getCdf(value); + } + + @ScalarFunction(value = "quantiles_at_values", visibility = EXPERIMENTAL) + @Description("For each input x between min/max values of t-digest, find which quantile is represented by that value") + @SqlType("array(double)") + public static Block quantilesAtValuesDouble(@SqlType("tdigest(double)") Slice input, @SqlType("array(double)") Block valuesArrayBlock) + { + TDigest tDigest = createTDigest(input); + BlockBuilder output = DOUBLE.createBlockBuilder(null, valuesArrayBlock.getPositionCount()); + for (int i = 0; i < valuesArrayBlock.getPositionCount(); i++) { + DOUBLE.writeDouble(output, tDigest.getCdf(DOUBLE.getDouble(valuesArrayBlock, i))); + } + return output.build(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/TryCastFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/TryCastFunction.java index 81464d91481ee..084d2d0eca417 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/TryCastFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/TryCastFunction.java @@ -21,6 +21,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableList; @@ -32,6 +33,7 @@ import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.metadata.CastType.CAST; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static java.lang.invoke.MethodHandles.catchException; import static java.lang.invoke.MethodHandles.constant; @@ -57,9 +59,9 @@ public TryCastFunction() } @Override - public boolean isHidden() + public final SqlFunctionVisibility getVisibility() { - return true; + return HIDDEN; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/TryFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/TryFunction.java index aefe0629b687c..2874900a7e1d1 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/TryFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/TryFunction.java @@ -30,9 +30,10 @@ import static com.facebook.presto.spi.StandardErrorCode.INVALID_CAST_ARGUMENT; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; import static com.facebook.presto.spi.StandardErrorCode.NUMERIC_VALUE_OUT_OF_RANGE; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; @Description("internal try function for desugaring TRY") -@ScalarFunction(value = "$internal$try", hidden = true, deterministic = false) +@ScalarFunction(value = "$internal$try", visibility = HIDDEN, deterministic = false) public final class TryFunction { private TryFunction() {} diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipFunction.java index 8621ce30e9e9c..a27ae65e1db39 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipFunction.java @@ -23,6 +23,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.RowType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -36,6 +37,7 @@ import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.util.Reflection.methodHandle; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -80,9 +82,9 @@ private ZipFunction(List typeParameters) } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return PUBLIC; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipWithFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipWithFunction.java index 55ae92ed6b42a..114475df98281 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipWithFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipWithFunction.java @@ -22,6 +22,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -36,6 +37,7 @@ import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.spi.type.TypeUtils.readNativeValue; import static com.facebook.presto.spi.type.TypeUtils.writeNativeValue; @@ -64,9 +66,9 @@ private ZipWithFunction() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return PUBLIC; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarImplementationHeader.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarImplementationHeader.java index dc7a3da110d5a..5afb77bd67343 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarImplementationHeader.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarImplementationHeader.java @@ -18,6 +18,7 @@ import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.ScalarFunction; import com.facebook.presto.spi.function.ScalarOperator; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.google.common.collect.ImmutableList; import java.lang.reflect.AnnotatedElement; @@ -27,6 +28,7 @@ import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.operator.annotations.FunctionsParserHelper.parseDescription; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; import static com.google.common.base.CaseFormat.LOWER_CAMEL; import static com.google.common.base.CaseFormat.LOWER_UNDERSCORE; import static com.google.common.base.Preconditions.checkArgument; @@ -79,15 +81,15 @@ public static List fromAnnotatedElement(AnnotatedEle if (scalarFunction != null) { String baseName = scalarFunction.value().isEmpty() ? camelToSnake(annotatedName(annotated)) : scalarFunction.value(); - builder.add(new ScalarImplementationHeader(baseName, new ScalarHeader(description, scalarFunction.hidden(), scalarFunction.deterministic(), scalarFunction.calledOnNullInput()))); + builder.add(new ScalarImplementationHeader(baseName, new ScalarHeader(description, scalarFunction.visibility(), scalarFunction.deterministic(), scalarFunction.calledOnNullInput()))); for (String alias : scalarFunction.alias()) { - builder.add(new ScalarImplementationHeader(alias, new ScalarHeader(description, scalarFunction.hidden(), scalarFunction.deterministic(), scalarFunction.calledOnNullInput()))); + builder.add(new ScalarImplementationHeader(alias, new ScalarHeader(description, scalarFunction.visibility(), scalarFunction.deterministic(), scalarFunction.calledOnNullInput()))); } } if (scalarOperator != null) { - builder.add(new ScalarImplementationHeader(scalarOperator.value(), new ScalarHeader(description, true, true, scalarOperator.value().isCalledOnNullInput()))); + builder.add(new ScalarImplementationHeader(scalarOperator.value(), new ScalarHeader(description, HIDDEN, true, scalarOperator.value().isCalledOnNullInput()))); } List result = builder.build(); @@ -110,9 +112,9 @@ public Optional getDescription() return header.getDescription(); } - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return header.isHidden(); + return header.getVisibility(); } public ScalarHeader getHeader() diff --git a/presto-main/src/main/java/com/facebook/presto/operator/window/SqlWindowFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/window/SqlWindowFunction.java index 4795c2e64674d..4a5dc4ab3750d 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/window/SqlWindowFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/window/SqlWindowFunction.java @@ -17,8 +17,10 @@ import com.facebook.presto.metadata.BuiltInFunction; import com.facebook.presto.metadata.FunctionManager; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.TypeManager; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static java.util.Objects.requireNonNull; public class SqlWindowFunction @@ -38,9 +40,9 @@ public final Signature getSignature() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return PUBLIC; } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java index e61da7adf0e50..2a00bc520332b 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java @@ -146,6 +146,7 @@ public class FeaturesConfig private Duration indexLoaderTimeout = new Duration(20, SECONDS); private boolean listBuiltInFunctionsOnly = true; + private boolean experimentalFunctionsEnabled; private PartitioningPrecisionStrategy partitioningPrecisionStrategy = PartitioningPrecisionStrategy.AUTOMATIC; @@ -1169,4 +1170,16 @@ public FeaturesConfig setPartitioningPrecisionStrategy(PartitioningPrecisionStra this.partitioningPrecisionStrategy = partitioningPrecisionStrategy; return this; } + + public boolean isExperimentalFunctionsEnabled() + { + return experimentalFunctionsEnabled; + } + + @Config("experimental-functions-enabled") + public FeaturesConfig setExperimentalFunctionsEnabled(boolean experimentalFunctionsEnabled) + { + this.experimentalFunctionsEnabled = experimentalFunctionsEnabled; + return this; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/type/BooleanOperators.java b/presto-main/src/main/java/com/facebook/presto/type/BooleanOperators.java index e684186460305..c1bf822e3b4ca 100644 --- a/presto-main/src/main/java/com/facebook/presto/type/BooleanOperators.java +++ b/presto-main/src/main/java/com/facebook/presto/type/BooleanOperators.java @@ -37,6 +37,7 @@ import static com.facebook.presto.spi.function.OperatorType.LESS_THAN; import static com.facebook.presto.spi.function.OperatorType.LESS_THAN_OR_EQUAL; import static com.facebook.presto.spi.function.OperatorType.NOT_EQUAL; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static java.lang.Float.floatToRawIntBits; import static java.nio.charset.StandardCharsets.US_ASCII; @@ -166,7 +167,7 @@ public static boolean indeterminate(@SqlType(StandardTypes.BOOLEAN) boolean valu } @SqlType(StandardTypes.BOOLEAN) - @ScalarFunction(hidden = true) // TODO: this should not be callable from SQL + @ScalarFunction(visibility = HIDDEN) // TODO: this should not be callable from SQL public static boolean not(@SqlType(StandardTypes.BOOLEAN) boolean value) { return !value; diff --git a/presto-main/src/main/java/com/facebook/presto/type/LikeFunctions.java b/presto-main/src/main/java/com/facebook/presto/type/LikeFunctions.java index 4e53313b38440..4aaad9b97f993 100644 --- a/presto-main/src/main/java/com/facebook/presto/type/LikeFunctions.java +++ b/presto-main/src/main/java/com/facebook/presto/type/LikeFunctions.java @@ -28,6 +28,7 @@ import io.airlift.slice.Slices; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; import static com.facebook.presto.spi.type.Chars.padSpaces; import static com.facebook.presto.util.Failures.checkCondition; import static io.airlift.joni.constants.MetaChar.INEFFECTIVE_META_CHAR; @@ -53,7 +54,7 @@ public final class LikeFunctions private LikeFunctions() {} - @ScalarFunction(value = "like", hidden = true) + @ScalarFunction(value = "like", visibility = HIDDEN) @LiteralParameters("x") @SqlType(StandardTypes.BOOLEAN) public static boolean likeChar(@LiteralParameter("x") Long x, @SqlType("char(x)") Slice value, @SqlType(LikePatternType.NAME) Regex pattern) @@ -62,7 +63,7 @@ public static boolean likeChar(@LiteralParameter("x") Long x, @SqlType("char(x)" } // TODO: this should not be callable from SQL - @ScalarFunction(value = "like", hidden = true) + @ScalarFunction(value = "like", visibility = HIDDEN) @LiteralParameters("x") @SqlType(StandardTypes.BOOLEAN) public static boolean likeVarchar(@SqlType("varchar(x)") Slice value, @SqlType(LikePatternType.NAME) Regex pattern) @@ -94,7 +95,7 @@ public static Regex likePattern(Slice pattern) return likePattern(pattern.toStringUtf8(), '0', false); } - @ScalarFunction(hidden = true) + @ScalarFunction(visibility = HIDDEN) @LiteralParameters({"x", "y"}) @SqlType(LikePatternType.NAME) public static Regex likePattern(@SqlType("varchar(x)") Slice pattern, @SqlType("varchar(y)") Slice escape) diff --git a/presto-main/src/main/java/com/facebook/presto/type/TDigestOperators.java b/presto-main/src/main/java/com/facebook/presto/type/TDigestOperators.java new file mode 100644 index 0000000000000..d29a20af11421 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/type/TDigestOperators.java @@ -0,0 +1,40 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.type; + +import com.facebook.presto.spi.function.ScalarOperator; +import com.facebook.presto.spi.function.SqlType; +import com.facebook.presto.spi.type.StandardTypes; +import io.airlift.slice.Slice; + +import static com.facebook.presto.spi.function.OperatorType.CAST; + +public final class TDigestOperators +{ + private TDigestOperators() {} + + @ScalarOperator(CAST) + @SqlType(StandardTypes.VARBINARY) + public static Slice castToBinaryDouble(@SqlType("tdigest(double)") Slice slice) + { + return slice; + } + + @ScalarOperator(CAST) + @SqlType("tdigest(double)") + public static Slice castFromVarbinaryDouble(@SqlType(StandardTypes.VARBINARY) Slice slice) + { + return slice; + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/type/TypeRegistry.java b/presto-main/src/main/java/com/facebook/presto/type/TypeRegistry.java index 9238c1f5da6d4..31550a54c90f8 100644 --- a/presto-main/src/main/java/com/facebook/presto/type/TypeRegistry.java +++ b/presto-main/src/main/java/com/facebook/presto/type/TypeRegistry.java @@ -64,6 +64,7 @@ import static com.facebook.presto.spi.type.RealType.REAL; import static com.facebook.presto.spi.type.RowType.Field; import static com.facebook.presto.spi.type.SmallintType.SMALLINT; +import static com.facebook.presto.spi.type.TDigestParametricType.TDIGEST; import static com.facebook.presto.spi.type.TimeType.TIME; import static com.facebook.presto.spi.type.TimeWithTimeZoneType.TIME_WITH_TIME_ZONE; import static com.facebook.presto.spi.type.TimestampType.TIMESTAMP; @@ -158,6 +159,7 @@ public TypeRegistry(Set types, FeaturesConfig featuresConfig) addParametricType(MAP); addParametricType(FUNCTION); addParametricType(QDIGEST); + addParametricType(TDIGEST); for (Type type : types) { addType(type); diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestFunctionManager.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestFunctionManager.java index bc0d8bd10cf02..4062e69b73264 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/TestFunctionManager.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestFunctionManager.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.metadata; +import com.facebook.presto.Session; import com.facebook.presto.block.BlockEncodingManager; import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; import com.facebook.presto.operator.scalar.CustomFunctions; @@ -23,6 +24,7 @@ import com.facebook.presto.spi.function.ScalarFunction; import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.function.SqlFunction; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.function.SqlType; import com.facebook.presto.spi.function.TypeVariableConstraint; import com.facebook.presto.spi.type.StandardTypes; @@ -40,6 +42,7 @@ import java.util.List; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; +import static com.facebook.presto.SystemSessionProperties.EXPERIMENTAL_FUNCTIONS_ENABLED; import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; @@ -47,6 +50,7 @@ import static com.facebook.presto.spi.function.OperatorType.SATURATED_FLOOR_CAST; import static com.facebook.presto.spi.function.OperatorType.tryGetOperatorType; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.HyperLogLogType.HYPER_LOG_LOG; import static com.facebook.presto.spi.type.TimestampWithTimeZoneType.TIMESTAMP_WITH_TIME_ZONE; @@ -55,6 +59,8 @@ import static com.facebook.presto.sql.planner.LiteralEncoder.getMagicLiteralFunctionSignature; import static com.facebook.presto.sql.tree.ArithmeticBinaryExpression.Operator.ADD; import static com.facebook.presto.sql.tree.ComparisonExpression.Operator.GREATER_THAN; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static com.facebook.presto.tpch.TpchMetadata.TINY_SCHEMA_NAME; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Lists.transform; import static java.lang.String.format; @@ -144,7 +150,7 @@ public void testConflictingScalarAggregation() } @Test - public void testListingHiddenFunctions() + public void testListingVisibilityBetaFunctionsDisabled() { TypeRegistry typeManager = new TypeRegistry(); FunctionManager functionManager = createFunctionManager(typeManager); @@ -154,6 +160,31 @@ public void testListingHiddenFunctions() assertTrue(names.contains("length"), "Expected function names " + names + " to contain 'length'"); assertTrue(names.contains("stddev"), "Expected function names " + names + " to contain 'stddev'"); assertTrue(names.contains("rank"), "Expected function names " + names + " to contain 'rank'"); + assertFalse(names.contains("tdigest_agg"), "Expected function names " + names + " not to contain 'tdigest_agg'"); + assertFalse(names.contains("quantiles_at_values"), "Expected function names " + names + " not to contain 'quantiles_at_values'"); + assertFalse(names.contains("like"), "Expected function names " + names + " not to contain 'like'"); + assertFalse(names.contains("$internal$sum_data_size_for_stats"), "Expected function names " + names + " not to contain '$internal$sum_data_size_for_stats'"); + assertFalse(names.contains("$internal$max_data_size_for_stats"), "Expected function names " + names + " not to contain '$internal$max_data_size_for_stats'"); + } + + @Test + public void testListingVisibilityBetaFunctionsEnabled() + { + Session session = testSessionBuilder() + .setCatalog("tpch") + .setSchema(TINY_SCHEMA_NAME) + .setSystemProperty(EXPERIMENTAL_FUNCTIONS_ENABLED, "true") + .build(); + TypeRegistry typeManager = new TypeRegistry(); + FunctionManager functionManager = createFunctionManager(typeManager); + List functions = functionManager.listFunctions(session); + List names = transform(functions, input -> input.getSignature().getNameSuffix()); + + assertTrue(names.contains("length"), "Expected function names " + names + " to contain 'length'"); + assertTrue(names.contains("stddev"), "Expected function names " + names + " to contain 'stddev'"); + assertTrue(names.contains("rank"), "Expected function names " + names + " to contain 'rank'"); + assertTrue(names.contains("tdigest_agg"), "Expected function names " + names + " to contain 'tdigest_agg'"); + assertTrue(names.contains("quantiles_at_values"), "Expected function names " + names + " to contain 'tdigest_agg'"); assertFalse(names.contains("like"), "Expected function names " + names + " not to contain 'like'"); assertFalse(names.contains("$internal$sum_data_size_for_stats"), "Expected function names " + names + " not to contain '$internal$sum_data_size_for_stats'"); assertFalse(names.contains("$internal$max_data_size_for_stats"), "Expected function names " + names + " not to contain '$internal$max_data_size_for_stats'"); @@ -441,9 +472,9 @@ public BuiltInScalarFunctionImplementation specialize( } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return PUBLIC; } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/GenericLongFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/GenericLongFunction.java index 4bbd8f694f337..b7bbb12577c1d 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/GenericLongFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/GenericLongFunction.java @@ -19,6 +19,7 @@ import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableList; @@ -29,6 +30,7 @@ import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; import static com.facebook.presto.spi.type.StandardTypes.BIGINT; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.util.Reflection.methodHandle; @@ -51,9 +53,9 @@ public final class GenericLongFunction } @Override - public boolean isHidden() + public final SqlFunctionVisibility getVisibility() { - return true; + return HIDDEN; } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForScalars.java b/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForScalars.java index 74841b28c84eb..41d065515ac58 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForScalars.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForScalars.java @@ -50,6 +50,9 @@ import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_BOXED_TYPE; import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.USE_NULL_FLAG; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.EXPERIMENTAL; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; @@ -89,7 +92,7 @@ public void testSingleImplementationScalarParse() assertEquals(scalar.getSignature(), expectedSignature); assertTrue(scalar.isDeterministic()); - assertFalse(scalar.isHidden()); + assertEquals(scalar.getVisibility(), PUBLIC); assertEquals(scalar.getDescription(), "Simple scalar with single implementation based on class"); assertImplementationCount(scalar, 1, 0, 0); @@ -100,8 +103,8 @@ public void testSingleImplementationScalarParse() assertEquals(specialized.getArgumentProperty(0).getNullConvention(), RETURN_NULL_ON_NULL); } - @ScalarFunction(value = "hidden_scalar_function", hidden = true) - @Description("Simple scalar with hidden property set") + @ScalarFunction(value = "hidden_scalar_function", visibility = HIDDEN) + @Description("Simple scalar with visibility set to hidden") public static class HiddenScalarFunction { @SqlType(StandardTypes.DOUBLE) @@ -111,6 +114,17 @@ public static double fun(@SqlType(StandardTypes.DOUBLE) double v) } } + @ScalarFunction(value = "beta_scalar_function", visibility = EXPERIMENTAL) + @Description("Simple scalar with visibility set to beta") + public static class BetaScalarFunction + { + @SqlType(StandardTypes.DOUBLE) + public static double fun(@SqlType(StandardTypes.DOUBLE) double v) + { + return v; + } + } + @Test public void testHiddenScalarParse() { @@ -119,7 +133,18 @@ public void testHiddenScalarParse() ParametricScalar scalar = (ParametricScalar) functions.get(0); assertTrue(scalar.isDeterministic()); - assertTrue(scalar.isHidden()); + assertEquals(scalar.getVisibility(), HIDDEN); + } + + @Test + public void testBetaScalarParse() + { + List functions = ScalarFromAnnotationsParser.parseFunctionDefinition(BetaScalarFunction.class); + assertEquals(functions.size(), 1); + ParametricScalar scalar = (ParametricScalar) functions.get(0); + + assertTrue(scalar.isDeterministic()); + assertEquals(scalar.getVisibility(), EXPERIMENTAL); } @ScalarFunction(value = "non_deterministic_scalar_function", deterministic = false) @@ -141,7 +166,7 @@ public void testNonDeterministicScalarParse() ParametricScalar scalar = (ParametricScalar) functions.get(0); assertFalse(scalar.isDeterministic()); - assertFalse(scalar.isHidden()); + assertEquals(scalar.getVisibility(), PUBLIC); } @ScalarFunction(value = "scalar_with_nullable", calledOnNullInput = true) @@ -173,7 +198,7 @@ public void testWithNullablePrimitiveArgScalarParse() assertEquals(scalar.getSignature(), expectedSignature); assertTrue(scalar.isDeterministic()); - assertFalse(scalar.isHidden()); + assertEquals(scalar.getVisibility(), PUBLIC); assertEquals(scalar.getDescription(), "Simple scalar with nullable primitive"); BuiltInScalarFunctionImplementation specialized = scalar.specialize(BoundVariables.builder().build(), 2, new TypeRegistry(), null); @@ -211,7 +236,7 @@ public void testWithNullableComplexArgScalarParse() assertEquals(scalar.getSignature(), expectedSignature); assertTrue(scalar.isDeterministic()); - assertFalse(scalar.isHidden()); + assertEquals(scalar.getVisibility(), PUBLIC); assertEquals(scalar.getDescription(), "Simple scalar with nullable complex type"); BuiltInScalarFunctionImplementation specialized = scalar.specialize(BoundVariables.builder().build(), 2, new TypeRegistry(), null); @@ -247,7 +272,7 @@ public void testStaticMethodScalarParse() assertEquals(scalar.getSignature(), expectedSignature); assertTrue(scalar.isDeterministic()); - assertFalse(scalar.isHidden()); + assertEquals(scalar.getVisibility(), PUBLIC); assertEquals(scalar.getDescription(), "Simple scalar with single implementation based on method"); } @@ -261,13 +286,21 @@ public static double fun1(@SqlType(StandardTypes.DOUBLE) double v) return v; } - @ScalarFunction(value = "static_method_scalar_2", hidden = true, deterministic = false) + @ScalarFunction(value = "static_method_scalar_2", visibility = HIDDEN, deterministic = false) @Description("Simple scalar with single implementation based on method 2") @SqlType(StandardTypes.BIGINT) public static long fun2(@SqlType(StandardTypes.BIGINT) long v) { return v; } + + @ScalarFunction(value = "static_method_scalar_3", visibility = EXPERIMENTAL, deterministic = false) + @Description("Simple scalar with single implementation based on method 3") + @SqlType(StandardTypes.BIGINT) + public static long fun3(@SqlType(StandardTypes.BIGINT) long v) + { + return v; + } } @Test @@ -285,23 +318,35 @@ public void testMultiScalarParse() BIGINT.getTypeSignature(), ImmutableList.of(BIGINT.getTypeSignature())); + Signature expectedSignature3 = new Signature( + QualifiedFunctionName.of(DEFAULT_NAMESPACE, "static_method_scalar_3"), + FunctionKind.SCALAR, + BIGINT.getTypeSignature(), + ImmutableList.of(BIGINT.getTypeSignature())); + List functions = ScalarFromAnnotationsParser.parseFunctionDefinitions(MultiScalarFunction.class); - assertEquals(functions.size(), 2); + assertEquals(functions.size(), 3); ParametricScalar scalar1 = (ParametricScalar) functions.stream().filter(signature -> signature.getSignature().equals(expectedSignature1)).collect(toImmutableList()).get(0); ParametricScalar scalar2 = (ParametricScalar) functions.stream().filter(signature -> signature.getSignature().equals(expectedSignature2)).collect(toImmutableList()).get(0); + ParametricScalar scalar3 = (ParametricScalar) functions.stream().filter(signature -> signature.getSignature().equals(expectedSignature3)).collect(toImmutableList()).get(0); assertImplementationCount(scalar1, 1, 0, 0); assertImplementationCount(scalar2, 1, 0, 0); assertEquals(scalar1.getSignature(), expectedSignature1); assertTrue(scalar1.isDeterministic()); - assertFalse(scalar1.isHidden()); + assertEquals(scalar1.getVisibility(), PUBLIC); assertEquals(scalar1.getDescription(), "Simple scalar with single implementation based on method 1"); assertEquals(scalar2.getSignature(), expectedSignature2); assertFalse(scalar2.isDeterministic()); - assertTrue(scalar2.isHidden()); + assertEquals(scalar2.getVisibility(), HIDDEN); assertEquals(scalar2.getDescription(), "Simple scalar with single implementation based on method 2"); + + assertEquals(scalar3.getSignature(), expectedSignature3); + assertFalse(scalar3.isDeterministic()); + assertEquals(scalar3.getVisibility(), EXPERIMENTAL); + assertEquals(scalar3.getDescription(), "Simple scalar with single implementation based on method 3"); } @ScalarFunction("parametric_scalar") @@ -342,7 +387,7 @@ public void testParametricScalarParse() assertEquals(scalar.getSignature(), expectedSignature); assertTrue(scalar.isDeterministic()); - assertFalse(scalar.isHidden()); + assertEquals(scalar.getVisibility(), PUBLIC); assertEquals(scalar.getDescription(), "Parametric scalar description"); } @@ -393,7 +438,7 @@ public void testComplexParametricScalarParse() assertEquals(scalar.getSignature(), expectedSignature); assertTrue(scalar.isDeterministic()); - assertFalse(scalar.isHidden()); + assertEquals(scalar.getVisibility(), PUBLIC); assertEquals(scalar.getDescription(), "Parametric scalar with exact and generic implementations"); } @@ -435,7 +480,7 @@ public void testSimpleInjectionScalarParse() assertEquals(scalar.getSignature(), expectedSignature); assertTrue(scalar.isDeterministic()); - assertFalse(scalar.isHidden()); + assertEquals(scalar.getVisibility(), PUBLIC); assertEquals(scalar.getDescription(), "Parametric scalar with literal injected"); } @@ -497,7 +542,7 @@ public void testConstructorInjectionScalarParse() assertEquals(scalar.getSignature(), expectedSignature); assertTrue(scalar.isDeterministic()); - assertFalse(scalar.isHidden()); + assertEquals(scalar.getVisibility(), PUBLIC); assertEquals(scalar.getDescription(), "Parametric scalar with type injected though constructor"); } @@ -533,7 +578,7 @@ public void testFixedTypeParameterParse() assertEquals(scalar.getSignature(), expectedSignature); assertTrue(scalar.isDeterministic()); - assertFalse(scalar.isHidden()); + assertEquals(scalar.getVisibility(), PUBLIC); assertEquals(scalar.getDescription(), "Parametric scalar that uses TypeParameter with fixed type"); } @@ -575,7 +620,7 @@ public void testPartiallyFixedTypeParameterParse() assertEquals(scalar.getSignature(), expectedSignature); assertTrue(scalar.isDeterministic()); - assertFalse(scalar.isHidden()); + assertEquals(scalar.getVisibility(), PUBLIC); assertEquals(scalar.getDescription(), "Parametric scalar that uses TypeParameter with partially fixed type"); } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/AbstractTestAggregationFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/AbstractTestAggregationFunction.java index 6bc7a5e556c97..6cb74770f7675 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/AbstractTestAggregationFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/AbstractTestAggregationFunction.java @@ -38,6 +38,7 @@ import static com.facebook.presto.operator.aggregation.AggregationTestUtils.assertAggregation; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypeSignatures; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static java.util.Objects.requireNonNull; public abstract class AbstractTestAggregationFunction { @@ -45,12 +46,21 @@ public abstract class AbstractTestAggregationFunction protected FunctionManager functionManager; protected Session session; + protected AbstractTestAggregationFunction() + { + this(testSessionBuilder().build()); + } + + protected AbstractTestAggregationFunction(Session session) + { + this.session = requireNonNull(session, "session is null"); + } + @BeforeClass public final void initTestAggregationFunction() { typeRegistry = new TypeRegistry(); functionManager = new FunctionManager(typeRegistry, new BlockEncodingManager(typeRegistry), new FeaturesConfig()); - session = testSessionBuilder().build(); } @AfterClass(alwaysRun = true) diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMergeQuantileDigestFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMergeQuantileDigestFunction.java index 5aed5095dbbe9..11e651292bccc 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMergeQuantileDigestFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMergeQuantileDigestFunction.java @@ -14,7 +14,6 @@ package com.facebook.presto.operator.aggregation; import com.facebook.airlift.stats.QuantileDigest; -import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.block.BlockBuilder; import com.facebook.presto.spi.type.DoubleType; @@ -22,18 +21,16 @@ import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeParameter; import com.google.common.collect.ImmutableList; -import org.testng.annotations.Test; import java.util.List; import java.util.function.BiFunction; -import static com.facebook.presto.operator.aggregation.AggregationTestUtils.assertAggregation; import static com.facebook.presto.spi.type.QuantileDigestParametricType.QDIGEST; import static io.airlift.slice.Slices.wrappedBuffer; import static java.util.Objects.requireNonNull; public class TestMergeQuantileDigestFunction - extends AbstractTestAggregationFunction + extends TestMergeStatisticalDigestFunction { public static final BiFunction QDIGEST_EQUALITY = (actualBinary, expectedBinary) -> { if (actualBinary == null && expectedBinary == null) { @@ -51,6 +48,12 @@ public class TestMergeQuantileDigestFunction actual.getMaxError() == expected.getMaxError(); }; + @Override + protected BiFunction getEquality() + { + return QDIGEST_EQUALITY; + } + @Override public Block[] getSequenceBlocks(int start, int length) { @@ -64,12 +67,6 @@ public Block[] getSequenceBlocks(int start, int length) return new Block[] {blockBuilder.build()}; } - @Override - protected String getFunctionName() - { - return "merge"; - } - @Override protected List getFunctionParameterTypes() { @@ -89,29 +86,4 @@ public Object getExpectedValue(int start, int length) } return new SqlVarbinary(qdigest.serialize().getBytes()); } - - // The following tests are overridden because by default simple equality checks are done, which often won't work with - // qdigests due to the way they are serialized. I am instead overridding these methods and using the QDIGEST_EQUALITY - // function to perform equality checks. - @Test - @Override - public void testMultiplePositions() - { - assertAggregation(getFunction(), - QDIGEST_EQUALITY, - "test multiple positions", - new Page(getSequenceBlocks(0, 5)), - getExpectedValue(0, 5)); - } - - @Test - @Override - public void testMixedNullAndNonNullPositions() - { - assertAggregation(getFunction(), - QDIGEST_EQUALITY, - "test mixed null and nonnull position", - new Page(createAlternatingNullsBlock(getFunction().getParameterTypes(), getSequenceBlocks(0, 10))), - getExpectedValueIncludingNulls(0, 10, 20)); - } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMergeStatisticalDigestFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMergeStatisticalDigestFunction.java new file mode 100644 index 0000000000000..8410fcdad96dd --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMergeStatisticalDigestFunction.java @@ -0,0 +1,70 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation; + +import com.facebook.presto.spi.Page; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.function.BiFunction; + +import static com.facebook.presto.operator.aggregation.AggregationTestUtils.assertAggregation; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; + +public abstract class TestMergeStatisticalDigestFunction + extends AbstractTestAggregationFunction +{ + protected TestMergeStatisticalDigestFunction() + { + super(testSessionBuilder().setSystemProperty("tdigest_enabled", "true").build()); + } + + protected abstract BiFunction getEquality(); + + @Override + protected String getFunctionName() + { + return "merge"; + } + + @Override + protected abstract List getFunctionParameterTypes(); + + @Override + public abstract Object getExpectedValue(int start, int length); + + // The following tests are overridden because by default simple equality checks are done, which often won't work with + // digests due to the way they are serialized. + @Test + @Override + public void testMultiplePositions() + { + assertAggregation(getFunction(), + getEquality(), + "test multiple positions", + new Page(getSequenceBlocks(0, 5)), + getExpectedValue(0, 5)); + } + + @Test + @Override + public void testMixedNullAndNonNullPositions() + { + assertAggregation(getFunction(), + getEquality(), + "test mixed null and nonnull position", + new Page(createAlternatingNullsBlock(getFunction().getParameterTypes(), getSequenceBlocks(0, 10))), + getExpectedValueIncludingNulls(0, 10, 20)); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMergeTDigestFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMergeTDigestFunction.java new file mode 100644 index 0000000000000..996ebec262bf1 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestMergeTDigestFunction.java @@ -0,0 +1,100 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation; + +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockBuilder; +import com.facebook.presto.spi.type.DoubleType; +import com.facebook.presto.spi.type.SqlVarbinary; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.spi.type.TypeParameter; +import com.facebook.presto.tdigest.TDigest; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.function.BiFunction; + +import static com.facebook.presto.spi.type.TDigestParametricType.TDIGEST; +import static com.facebook.presto.tdigest.TDigest.createTDigest; +import static io.airlift.slice.Slices.wrappedBuffer; +import static java.lang.Math.abs; +import static java.lang.Math.max; +import static java.util.Objects.requireNonNull; + +public class TestMergeTDigestFunction + extends TestMergeStatisticalDigestFunction +{ + private static final double[] quantiles = {0.01, 0.05, 0.1, 0.25, 0.50, 0.75, 0.9, 0.95, 0.99}; + + public static final BiFunction TDIGEST_EQUALITY = (actualBinary, expectedBinary) -> { + if (actualBinary == null && expectedBinary == null) { + return true; + } + requireNonNull(actualBinary, "actual value was null"); + requireNonNull(expectedBinary, "expected value was null"); + + TDigest actual = createTDigest(wrappedBuffer(((SqlVarbinary) actualBinary).getBytes())); + TDigest expected = createTDigest(wrappedBuffer(((SqlVarbinary) expectedBinary).getBytes())); + + for (double quantile : quantiles) { + if (abs(actual.getQuantile(quantile) - expected.getQuantile(quantile)) > max(0.5, abs(actual.getQuantile(quantile) * 0.01))) { + return false; + } + } + + return actual.getSize() == expected.getSize() && + actual.getMin() == expected.getMin() && + actual.getMax() == expected.getMax() && + actual.getCompressionFactor() == expected.getCompressionFactor(); + }; + + @Override + protected BiFunction getEquality() + { + return TDIGEST_EQUALITY; + } + + @Override + public Block[] getSequenceBlocks(int start, int length) + { + Type type = TDIGEST.createType(typeRegistry, ImmutableList.of(TypeParameter.of(DoubleType.DOUBLE))); + BlockBuilder blockBuilder = type.createBlockBuilder(null, length); + for (int i = start; i < start + length; i++) { + TDigest tdigest = createTDigest(100); + tdigest.add(i); + type.writeSlice(blockBuilder, tdigest.serialize()); + } + return new Block[] {blockBuilder.build()}; + } + + @Override + protected List getFunctionParameterTypes() + { + return ImmutableList.of("tdigest(double)"); + } + + @Override + public Object getExpectedValue(int start, int length) + { + if (length == 0) { + return null; + } + + TDigest tdigest = createTDigest(100); + for (int i = start; i < start + length; i++) { + tdigest.add(i); + } + return new SqlVarbinary(tdigest.serialize().getBytes()); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestQuantileDigestAggregationFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestQuantileDigestAggregationFunction.java index 9c8f95fc52c97..9512bee67a6f1 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestQuantileDigestAggregationFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestQuantileDigestAggregationFunction.java @@ -15,26 +15,21 @@ import com.facebook.airlift.stats.QuantileDigest; import com.facebook.presto.metadata.FunctionManager; -import com.facebook.presto.metadata.MetadataManager; -import com.facebook.presto.operator.scalar.AbstractTestFunctions; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.block.Block; import com.facebook.presto.spi.type.SqlVarbinary; import com.facebook.presto.spi.type.StandardTypes; import com.facebook.presto.spi.type.Type; -import com.google.common.base.Joiner; +import com.google.common.primitives.Doubles; import com.google.common.primitives.Floats; import org.testng.annotations.Test; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import java.util.stream.LongStream; import static com.facebook.presto.block.BlockAssertions.createBlockOfReals; -import static com.facebook.presto.block.BlockAssertions.createDoubleSequenceBlock; -import static com.facebook.presto.block.BlockAssertions.createDoublesBlock; import static com.facebook.presto.block.BlockAssertions.createLongSequenceBlock; import static com.facebook.presto.block.BlockAssertions.createLongsBlock; import static com.facebook.presto.block.BlockAssertions.createRLEBlock; @@ -44,55 +39,21 @@ import static com.facebook.presto.operator.aggregation.FloatingPointBitsConverterUtil.floatToSortableInt; import static com.facebook.presto.operator.aggregation.TestMergeQuantileDigestFunction.QDIGEST_EQUALITY; import static com.facebook.presto.spi.type.BigintType.BIGINT; -import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; import static com.facebook.presto.spi.type.RealType.REAL; -import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; +import static com.facebook.presto.spi.type.StandardTypes.QDIGEST; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; import static com.google.common.collect.ImmutableList.toImmutableList; import static java.lang.Double.NaN; -import static java.lang.Integer.max; -import static java.lang.Integer.min; -import static java.lang.String.format; public class TestQuantileDigestAggregationFunction - extends AbstractTestFunctions + extends TestStatisticalDigestAggregationFunction { - private static final Joiner ARRAY_JOINER = Joiner.on(","); - private static final MetadataManager METADATA = MetadataManager.createTestMetadataManager(); + private static final double STANDARD_ERROR = 0.01; - @Test - public void testDoublesWithWeights() + protected double getParameter() { - testAggregationDouble( - createDoublesBlock(1.0, null, 2.0, null, 3.0, null, 4.0, null, 5.0, null), - createRLEBlock(1, 10), - 0.01, 1.0, 2.0, 3.0, 4.0, 5.0); - testAggregationDouble( - createDoublesBlock(null, null, null, null, null), - createRLEBlock(1, 5), - NaN); - testAggregationDouble( - createDoublesBlock(-1.0, -2.0, -3.0, -4.0, -5.0, -6.0, -7.0, -8.0, -9.0, -10.0), - createRLEBlock(1, 10), - 0.01, -1.0, -2.0, -3.0, -4.0, -5.0, -6.0, -7.0, -8.0, -9.0, -10.0); - testAggregationDouble( - createDoublesBlock(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0), - createRLEBlock(1, 10), - 0.01, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0); - testAggregationDouble( - createDoublesBlock(), - createRLEBlock(1, 0), - NaN); - testAggregationDouble( - createDoublesBlock(1.0), - createRLEBlock(1, 1), - 0.01, 1.0); - testAggregationDouble( - createDoubleSequenceBlock(-1000, 1000), - createRLEBlock(1, 2000), - 0.01, - LongStream.range(-1000, 1000).asDoubleStream().toArray()); + return STANDARD_ERROR; } @Test @@ -163,7 +124,8 @@ public void testBigintsWithWeight() LongStream.range(-1000, 1000).toArray()); } - private InternalAggregationFunction getAggregationFunction(Type... type) + @Override + protected InternalAggregationFunction getAggregationFunction(Type... type) { FunctionManager functionManager = METADATA.getFunctionManager(); return functionManager.getAggregateFunctionImplementation( @@ -215,28 +177,6 @@ private void testAggregationReal(Block longsBlock, Block weightsBlock, double ma inputs); } - private void testAggregationDouble(Block longsBlock, Block weightsBlock, double maxError, double... inputs) - { - // Test without weights and accuracy - testAggregationDoubles( - getAggregationFunction(DOUBLE), - new Page(longsBlock), - maxError, - inputs); - // Test with weights and without accuracy - testAggregationDoubles( - getAggregationFunction(DOUBLE, BIGINT), - new Page(longsBlock, weightsBlock), - maxError, - inputs); - // Test with weights and accuracy - testAggregationDoubles( - getAggregationFunction(DOUBLE, BIGINT, DOUBLE), - new Page(longsBlock, weightsBlock, createRLEBlock(maxError, longsBlock.getPositionCount())), - maxError, - inputs); - } - private void testAggregationBigints(InternalAggregationFunction function, Page page, double maxError, long... inputs) { // aggregate level @@ -250,37 +190,38 @@ private void testAggregationBigints(InternalAggregationFunction function, Page p List rows = Arrays.stream(inputs).sorted().boxed().collect(Collectors.toList()); SqlVarbinary returned = (SqlVarbinary) AggregationTestUtils.aggregation(function, page); - assertPercentileWithinError(StandardTypes.BIGINT, returned, maxError, rows, 0.1, 0.5, 0.9, 0.99); + assertPercentileWithinError(QDIGEST, StandardTypes.BIGINT, returned, maxError, rows, 0.1, 0.5, 0.9, 0.99); } - private void testAggregationDoubles(InternalAggregationFunction function, Page page, double maxError, double... inputs) + private void testAggregationReal(InternalAggregationFunction function, Page page, double maxError, float... inputs) { assertAggregation(function, QDIGEST_EQUALITY, "test multiple positions", page, - getExpectedValueDoubles(maxError, inputs)); + getExpectedValuesFloats(maxError, inputs)); // test scalars - List rows = Arrays.stream(inputs).sorted().boxed().collect(Collectors.toList()); + List rows = Floats.asList(inputs).stream().sorted().map(Float::doubleValue).collect(Collectors.toList()); SqlVarbinary returned = (SqlVarbinary) AggregationTestUtils.aggregation(function, page); - assertPercentileWithinError(StandardTypes.DOUBLE, returned, maxError, rows, 0.1, 0.5, 0.9, 0.99); + assertPercentileWithinError(QDIGEST, StandardTypes.REAL, returned, maxError, rows, 0.1, 0.5, 0.9, 0.99); } - private void testAggregationReal(InternalAggregationFunction function, Page page, double maxError, float... inputs) + @Override + protected void testAggregationDoubles(InternalAggregationFunction function, Page page, double maxError, double... inputs) { assertAggregation(function, QDIGEST_EQUALITY, "test multiple positions", page, - getExpectedValuesFloats(maxError, inputs)); + getExpectedValueDoubles(maxError, inputs)); // test scalars - List rows = Floats.asList(inputs).stream().sorted().map(Float::doubleValue).collect(Collectors.toList()); + List rows = Doubles.asList(inputs).stream().sorted().collect(Collectors.toList()); SqlVarbinary returned = (SqlVarbinary) AggregationTestUtils.aggregation(function, page); - assertPercentileWithinError(StandardTypes.REAL, returned, maxError, rows, 0.1, 0.5, 0.9, 0.99); + assertPercentileWithinError(QDIGEST, StandardTypes.DOUBLE, returned, maxError, rows, 0.1, 0.5, 0.9, 0.99); } private Object getExpectedValueLongs(double maxError, long... values) @@ -293,7 +234,8 @@ private Object getExpectedValueLongs(double maxError, long... values) return new SqlVarbinary(qdigest.serialize().getBytes()); } - private Object getExpectedValueDoubles(double maxError, double... values) + @Override + protected Object getExpectedValueDoubles(double maxError, double... values) { if (values.length == 0) { return null; @@ -312,79 +254,4 @@ private Object getExpectedValuesFloats(double maxError, float... values) Floats.asList(values).forEach(value -> qdigest.add(floatToSortableInt(value))); return new SqlVarbinary(qdigest.serialize().getBytes()); } - - private void assertPercentileWithinError(String type, SqlVarbinary binary, double error, List rows, double... percentiles) - { - if (rows.isEmpty()) { - // Nothing to assert except that the qdigest is empty - return; - } - - // Test each quantile individually (value_at_quantile) - for (double percentile : percentiles) { - assertPercentileWithinError(type, binary, error, rows, percentile); - } - - // Test all the quantiles (values_at_quantiles) - assertPercentilesWithinError(type, binary, error, rows, percentiles); - } - - private void assertPercentileWithinError(String type, SqlVarbinary binary, double error, List rows, double percentile) - { - Number lowerBound = getLowerBound(error, rows, percentile); - Number upperBound = getUpperBound(error, rows, percentile); - - // Check that the chosen quantile is within the upper and lower bound of the error - functionAssertions.assertFunction( - format("value_at_quantile(CAST(X'%s' AS qdigest(%s)), %s) >= %s", binary.toString().replaceAll("\\s+", " "), type, percentile, lowerBound), - BOOLEAN, - true); - functionAssertions.assertFunction( - format("value_at_quantile(CAST(X'%s' AS qdigest(%s)), %s) <= %s", binary.toString().replaceAll("\\s+", " "), type, percentile, upperBound), - BOOLEAN, - true); - } - - private void assertPercentilesWithinError(String type, SqlVarbinary binary, double error, List rows, double[] percentiles) - { - List boxedPercentiles = Arrays.stream(percentiles).sorted().boxed().collect(toImmutableList()); - List lowerBounds = boxedPercentiles.stream().map(percentile -> getLowerBound(error, rows, percentile)).collect(toImmutableList()); - List upperBounds = boxedPercentiles.stream().map(percentile -> getUpperBound(error, rows, percentile)).collect(toImmutableList()); - - // Ensure that the lower bound of each item in the distribution is not greater than the chosen quantiles - functionAssertions.assertFunction( - format( - "zip_with(values_at_quantiles(CAST(X'%s' AS qdigest(%s)), ARRAY[%s]), ARRAY[%s], (value, lowerbound) -> value >= lowerbound)", - binary.toString().replaceAll("\\s+", " "), - type, - ARRAY_JOINER.join(boxedPercentiles), - ARRAY_JOINER.join(lowerBounds)), - METADATA.getType(parseTypeSignature("array(boolean)")), - Collections.nCopies(percentiles.length, true)); - - // Ensure that the upper bound of each item in the distribution is not less than the chosen quantiles - functionAssertions.assertFunction( - format( - "zip_with(values_at_quantiles(CAST(X'%s' AS qdigest(%s)), ARRAY[%s]), ARRAY[%s], (value, upperbound) -> value <= upperbound)", - binary.toString().replaceAll("\\s+", " "), - type, - ARRAY_JOINER.join(boxedPercentiles), - ARRAY_JOINER.join(upperBounds)), - METADATA.getType(parseTypeSignature("array(boolean)")), - Collections.nCopies(percentiles.length, true)); - } - - private Number getLowerBound(double error, List rows, double percentile) - { - int medianIndex = (int) (rows.size() * percentile); - int marginOfError = (int) (rows.size() * error / 2); - return rows.get(max(medianIndex - marginOfError, 0)); - } - - private Number getUpperBound(double error, List rows, double percentile) - { - int medianIndex = (int) (rows.size() * percentile); - int marginOfError = (int) (rows.size() * error / 2); - return rows.get(min(medianIndex + marginOfError, rows.size() - 1)); - } } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestStatisticalDigestAggregationFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestStatisticalDigestAggregationFunction.java new file mode 100644 index 0000000000000..7a47f8f63f538 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestStatisticalDigestAggregationFunction.java @@ -0,0 +1,195 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation; + +import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.operator.scalar.AbstractTestFunctions; +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.type.SqlVarbinary; +import com.facebook.presto.spi.type.Type; +import com.google.common.base.Joiner; +import org.testng.annotations.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.LongStream; + +import static com.facebook.presto.block.BlockAssertions.createDoubleSequenceBlock; +import static com.facebook.presto.block.BlockAssertions.createDoublesBlock; +import static com.facebook.presto.block.BlockAssertions.createRLEBlock; +import static com.facebook.presto.spi.type.BigintType.BIGINT; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.lang.Double.NaN; +import static java.lang.Integer.max; +import static java.lang.Integer.min; +import static java.lang.String.format; + +public abstract class TestStatisticalDigestAggregationFunction + extends AbstractTestFunctions +{ + static final Joiner ARRAY_JOINER = Joiner.on(","); + protected static final MetadataManager METADATA = MetadataManager.createTestMetadataManager(); + + @Test + public void testDoublesWithWeights() + { + testAggregationDouble( + createDoublesBlock(1.0, null, 2.0, null, 3.0, null, 4.0, null, 5.0, null), + createRLEBlock(1, 10), + getParameter(), + 1.0, 2.0, 3.0, 4.0, 5.0); + testAggregationDouble( + createDoublesBlock(null, null, null, null, null), + createRLEBlock(1, 5), + NaN); + testAggregationDouble( + createDoublesBlock(-1.0, -2.0, -3.0, -4.0, -5.0, -6.0, -7.0, -8.0, -9.0, -10.0), + createRLEBlock(1, 10), + getParameter(), + -1.0, -2.0, -3.0, -4.0, -5.0, -6.0, -7.0, -8.0, -9.0, -10.0); + testAggregationDouble( + createDoublesBlock(1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0), + createRLEBlock(1, 10), + getParameter(), + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0); + testAggregationDouble( + createDoublesBlock(), + createRLEBlock(1, 0), + NaN); + testAggregationDouble( + createDoublesBlock(1.0), + createRLEBlock(1, 1), + getParameter(), + 1.0); + testAggregationDouble( + createDoubleSequenceBlock(-1000, 1000), + createRLEBlock(1, 2000), + getParameter(), + LongStream.range(-1000, 1000).asDoubleStream().toArray()); + } + + protected abstract InternalAggregationFunction getAggregationFunction(Type... type); + + private void testAggregationDouble(Block longsBlock, Block weightsBlock, double parameter, double... inputs) + { + // Test without weights and accuracy + testAggregationDoubles( + getAggregationFunction(DOUBLE), + new Page(longsBlock), + parameter, + inputs); + // Test with weights and without accuracy + testAggregationDoubles( + getAggregationFunction(DOUBLE, BIGINT), + new Page(longsBlock, weightsBlock), + parameter, + inputs); + // Test with weights and accuracy + testAggregationDoubles( + getAggregationFunction(DOUBLE, BIGINT, DOUBLE), + new Page(longsBlock, weightsBlock, createRLEBlock(parameter, longsBlock.getPositionCount())), + parameter, + inputs); + } + + abstract double getParameter(); + + abstract void testAggregationDoubles(InternalAggregationFunction function, Page page, double maxError, double... inputs); + + abstract Object getExpectedValueDoubles(double maxError, double... values); + + void assertPercentileWithinError(String method, String type, SqlVarbinary binary, double error, List rows, double... percentiles) + { + if (rows.isEmpty()) { + // Nothing to assert except that the digest is empty + return; + } + + // Test each quantile individually (value_at_quantile) + for (double percentile : percentiles) { + assertPercentileWithinError(method, type, binary, error, rows, percentile); + } + + // Test all the quantiles (values_at_quantiles) + assertPercentilesWithinError(method, type, binary, error, rows, percentiles); + } + + private void assertPercentileWithinError(String method, String type, SqlVarbinary binary, double error, List rows, double percentile) + { + Number lowerBound = getLowerBoundValue(error, rows, percentile); + Number upperBound = getUpperBoundValue(error, rows, percentile); + + // Check that the chosen quantile is within the upper and lower bound of the error + functionAssertions.assertFunction( + format("value_at_quantile(CAST(X'%s' AS %s(%s)), %s) >= %s", binary.toString().replaceAll("\\s+", " "), method, + type, percentile, lowerBound), + BOOLEAN, + true); + functionAssertions.assertFunction( + format("value_at_quantile(CAST(X'%s' AS %s(%s)), %s) <= %s", binary.toString().replaceAll("\\s+", " "), + method, type, percentile, upperBound), + BOOLEAN, + true); + } + + private void assertPercentilesWithinError(String method, String type, SqlVarbinary binary, double error, List rows, double[] percentiles) + { + List boxedPercentiles = Arrays.stream(percentiles).sorted().boxed().collect(toImmutableList()); + List lowerBounds = boxedPercentiles.stream().map(percentile -> getLowerBoundValue(error, rows, percentile)).collect(toImmutableList()); + List upperBounds = boxedPercentiles.stream().map(percentile -> getUpperBoundValue(error, rows, percentile)).collect(toImmutableList()); + + // Ensure that the lower bound of each item in the distribution is not greater than the chosen quantiles + functionAssertions.assertFunction( + format( + "zip_with(values_at_quantiles(CAST(X'%s' AS %s(%s)), ARRAY[%s]), ARRAY[%s], (value, lowerbound) -> value >= lowerbound)", + binary.toString().replaceAll("\\s+", " "), + method, + type, + ARRAY_JOINER.join(boxedPercentiles), + ARRAY_JOINER.join(lowerBounds)), + METADATA.getType(parseTypeSignature("array(boolean)")), + Collections.nCopies(percentiles.length, true)); + + // Ensure that the upper bound of each item in the distribution is not less than the chosen quantiles + functionAssertions.assertFunction( + format( + "zip_with(values_at_quantiles(CAST(X'%s' AS %s(%s)), ARRAY[%s]), ARRAY[%s], (value, upperbound) -> value <= upperbound)", + binary.toString().replaceAll("\\s+", " "), + method, + type, + ARRAY_JOINER.join(boxedPercentiles), + ARRAY_JOINER.join(upperBounds)), + METADATA.getType(parseTypeSignature("array(boolean)")), + Collections.nCopies(percentiles.length, true)); + } + + private Number getLowerBoundValue(double error, List rows, double percentile) + { + int medianIndex = (int) (rows.size() * percentile); + int marginOfError = (int) (rows.size() * error / 2); + return rows.get(max(medianIndex - marginOfError, 0)); + } + + private Number getUpperBoundValue(double error, List rows, double percentile) + { + int medianIndex = (int) (rows.size() * percentile); + int marginOfError = (int) (rows.size() * error / 2); + return rows.get(min(medianIndex + marginOfError, rows.size() - 1)); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestTDigestAggregationFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestTDigestAggregationFunction.java new file mode 100644 index 0000000000000..50c22f579a563 --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/aggregation/TestTDigestAggregationFunction.java @@ -0,0 +1,181 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.aggregation; + +import com.facebook.presto.metadata.FunctionManager; +import com.facebook.presto.spi.Page; +import com.facebook.presto.spi.type.SqlVarbinary; +import com.facebook.presto.spi.type.StandardTypes; +import com.facebook.presto.spi.type.Type; +import com.facebook.presto.tdigest.TDigest; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +import static com.facebook.presto.operator.aggregation.AggregationTestUtils.assertAggregation; +import static com.facebook.presto.operator.aggregation.TestMergeTDigestFunction.TDIGEST_EQUALITY; +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.StandardTypes.TDIGEST; +import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; +import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; +import static com.facebook.presto.tdigest.TDigest.createTDigest; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.lang.String.format; +import static java.util.Collections.sort; + +public class TestTDigestAggregationFunction + extends TestStatisticalDigestAggregationFunction +{ + private static final double STANDARD_COMPRESSION_FACTOR = 100; + + protected double getParameter() + { + return STANDARD_COMPRESSION_FACTOR; + } + + @Override + protected InternalAggregationFunction getAggregationFunction(Type... type) + { + FunctionManager functionManager = METADATA.getFunctionManager(); + return functionManager.getAggregateFunctionImplementation( + functionManager.lookupFunction("tdigest_agg", fromTypes(type))); + } + + @Override + protected void testAggregationDoubles(InternalAggregationFunction function, Page page, double error, double... inputs) + { + assertAggregation(function, + TDIGEST_EQUALITY, + "test multiple positions", + page, + getExpectedValueDoubles(STANDARD_COMPRESSION_FACTOR, inputs)); + + // test scalars + List rows = Arrays.stream(inputs).sorted().boxed().collect(Collectors.toList()); + + SqlVarbinary returned = (SqlVarbinary) AggregationTestUtils.aggregation(function, page); + assertPercentileWithinError(TDIGEST, StandardTypes.DOUBLE, returned, 0.01, rows, 0.1, 0.5, 0.9, 0.99); + assertValueWithinError(StandardTypes.DOUBLE, returned, STANDARD_COMPRESSION_FACTOR, rows, 0.1, 0.5, 0.9, 0.99); + } + + @Override + protected Object getExpectedValueDoubles(double compression, double... values) + { + if (values.length == 0) { + return null; + } + TDigest tDigest = createTDigest(compression); + Arrays.stream(values).forEach(tDigest::add); + return new SqlVarbinary(tDigest.serialize().getBytes()); + } + + private void assertValueWithinError(String type, SqlVarbinary binary, double error, List rows, double... percentiles) + { + if (rows.isEmpty()) { + // Nothing to assert except that the tdigest is empty + return; + } + + // Test each quantile individually (value_at_quantile) + for (double percentile : percentiles) { + assertValueWithinError(type, binary, error, rows, percentile); + } + + // Test all the quantiles (values_at_quantiles) + assertValuesWithinError(type, binary, error, rows, percentiles); + } + + private void assertValueWithinError(String type, SqlVarbinary binary, double error, List rows, double percentile) + { + Number lowerBound = getLowerBoundQuantile(percentile, error); + Number upperBound = getUpperBoundQuantile(percentile, error); + + // Check that the chosen quantile is within the upper and lower bound of the error + functionAssertions.assertFunction( + format("quantile_at_value(CAST(X'%s' AS tdigest(%s)), %s) >= %s", + binary.toString().replaceAll("\\s+", " "), + type, + sortNumberList(rows).get((int) (rows.size() * percentile)).doubleValue(), + lowerBound), + BOOLEAN, + true); + functionAssertions.assertFunction( + format("quantile_at_value(CAST(X'%s' AS tdigest(%s)), %s) <= %s", + binary.toString().replaceAll("\\s+", " "), + type, + sortNumberList(rows).get((int) (rows.size() * percentile)).doubleValue(), + upperBound), + BOOLEAN, + true); + } + + private void assertValuesWithinError(String type, SqlVarbinary binary, double error, List rows, double[] percentiles) + { + List boxedPercentiles = Arrays.stream(percentiles).sorted().boxed().collect(toImmutableList()); + List boxedValues = boxedPercentiles.stream().map(percentile -> sortNumberList(rows).get((int) (rows.size() * percentile)).doubleValue()).collect(toImmutableList()); + List lowerBounds = boxedPercentiles.stream().map(percentile -> getLowerBoundQuantile(percentile, error)).collect(toImmutableList()); + List upperBounds = boxedPercentiles.stream().map(percentile -> getUpperBoundQuantile(percentile, error)).collect(toImmutableList()); + + // Ensure that the lower bound of each item in the distribution is not greater than the chosen quantiles + functionAssertions.assertFunction( + format( + "zip_with(quantiles_at_values(CAST(X'%s' AS tdigest(%s)), ARRAY[%s]), ARRAY[%s], (value, lowerbound) -> value >= lowerbound)", + binary.toString().replaceAll("\\s+", " "), + type, + ARRAY_JOINER.join(boxedValues), + ARRAY_JOINER.join(lowerBounds)), + METADATA.getType(parseTypeSignature("array(boolean)")), + Collections.nCopies(percentiles.length, true)); + + // Ensure that the upper bound of each item in the distribution is not less than the chosen quantiles + functionAssertions.assertFunction( + format( + "zip_with(quantiles_at_values(CAST(X'%s' AS tdigest(%s)), ARRAY[%s]), ARRAY[%s], (value, upperbound) -> value <= upperbound)", + binary.toString().replaceAll("\\s+", " "), + type, + ARRAY_JOINER.join(boxedValues), + ARRAY_JOINER.join(upperBounds)), + METADATA.getType(parseTypeSignature("array(boolean)")), + Collections.nCopies(percentiles.length, true)); + } + + private Number getLowerBoundQuantile(double quantile, double error) + { + return Math.max(0, quantile - error); + } + + private Number getUpperBoundQuantile(double quantile, double error) + { + return Math.min(1, quantile + error); + } + + private List sortNumberList(List list) + { + List sorted = new ArrayList<>(list); + sort(sorted, new Comparator() { + @Override + public int compare(Number o1, Number o2) + { + Double d1 = (o1 == null) ? Double.POSITIVE_INFINITY : o1.doubleValue(); + Double d2 = (o2 == null) ? Double.POSITIVE_INFINITY : o2.doubleValue(); + return d1.compareTo(d2); + } + }); + return sorted; + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayFilter.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayFilter.java index 60728c9036f3a..4e12481afe1d4 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayFilter.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayFilter.java @@ -27,6 +27,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.LambdaDefinitionExpression; import com.facebook.presto.spi.relation.RowExpression; @@ -68,6 +69,7 @@ import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.OperatorType.GREATER_THAN; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static com.facebook.presto.spi.type.BigintType.BIGINT; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; @@ -213,9 +215,9 @@ private ExactArrayFilterFunction() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return PUBLIC; } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestProvidedBlockBuilderReturnPlaceConvention.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestProvidedBlockBuilderReturnPlaceConvention.java index 69bc83c23b345..0e17cbbdc69c3 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestProvidedBlockBuilderReturnPlaceConvention.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestProvidedBlockBuilderReturnPlaceConvention.java @@ -23,6 +23,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.ArrayType; import com.facebook.presto.spi.type.Type; import com.facebook.presto.spi.type.TypeManager; @@ -44,6 +45,7 @@ import static com.facebook.presto.operator.scalar.TestProvidedBlockBuilderReturnPlaceConvention.FunctionWithProvidedBlockReturnPlaceConvention1.PROVIDED_BLOCKBUILDER_CONVENTION1; import static com.facebook.presto.operator.scalar.TestProvidedBlockBuilderReturnPlaceConvention.FunctionWithProvidedBlockReturnPlaceConvention2.PROVIDED_BLOCKBUILDER_CONVENTION2; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; import static com.facebook.presto.spi.type.DoubleType.DOUBLE; import static com.facebook.presto.spi.type.IntegerType.INTEGER; @@ -242,9 +244,9 @@ public boolean isDeterministic() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return PUBLIC; } @Override @@ -387,9 +389,9 @@ public boolean isDeterministic() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return PUBLIC; } @Override diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestTDigestFunctions.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestTDigestFunctions.java new file mode 100644 index 0000000000000..e688a1669c89d --- /dev/null +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestTDigestFunctions.java @@ -0,0 +1,611 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.operator.scalar; + +import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.spi.type.SqlVarbinary; +import com.facebook.presto.tdigest.TDigest; +import com.google.common.base.Joiner; +import org.apache.commons.math3.distribution.BinomialDistribution; +import org.apache.commons.math3.distribution.GeometricDistribution; +import org.apache.commons.math3.distribution.NormalDistribution; +import org.apache.commons.math3.distribution.PoissonDistribution; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static com.facebook.presto.spi.type.BooleanType.BOOLEAN; +import static com.facebook.presto.spi.type.DoubleType.DOUBLE; +import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; +import static com.facebook.presto.tdigest.TDigest.createTDigest; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.lang.String.format; +import static java.util.Collections.sort; + +public class TestTDigestFunctions + extends AbstractTestFunctions +{ + private static final int NUMBER_OF_ENTRIES = 1_000_000; + private static final int STANDARD_COMPRESSION_FACTOR = 100; + private static final double STANDARD_ERROR = 0.01; + private static final double[] quantiles = {0.0001, 0.0200, 0.0300, 0.04000, 0.0500, 0.1000, 0.2000, 0.3000, 0.4000, 0.5000, 0.6000, 0.7000, 0.8000, + 0.9000, 0.9500, 0.9600, 0.9700, 0.9800, 0.9999}; + + private static final Joiner ARRAY_JOINER = Joiner.on(","); + private static final MetadataManager METADATA = MetadataManager.createTestMetadataManager(); + + @Test + public void testNullTDigestGetValueAtQuantile() + { + functionAssertions.assertFunction("value_at_quantile(CAST(NULL AS tdigest(double)), 0.3)", DOUBLE, null); + } + + @Test + public void testNullTDigestGetQuantileAtValue() + { + functionAssertions.assertFunction("quantile_at_value(CAST(NULL AS tdigest(double)), 0.3)", DOUBLE, null); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testGetValueAtQuantileOverOne() + { + functionAssertions.assertFunction(format("value_at_quantile(CAST(X'%s' AS tdigest(double)), 1.5)", + new SqlVarbinary(createTDigest(STANDARD_COMPRESSION_FACTOR).serialize().getBytes()).toString().replaceAll("\\s+", " ")), + DOUBLE, + null); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testGetValueAtQuantileBelowZero() + { + functionAssertions.assertFunction(format("value_at_quantile(CAST(X'%s' AS tdigest(double)), -0.2)", + new SqlVarbinary(createTDigest(STANDARD_COMPRESSION_FACTOR).serialize().getBytes()).toString().replaceAll("\\s+", " ")), + DOUBLE, + null); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testInvalidSerializationFormat() + { + functionAssertions.assertFunction(format("value_at_quantile(CAST(X'%s' AS tdigest(double)), 0.5)", + new SqlVarbinary(createTDigest(STANDARD_COMPRESSION_FACTOR).serialize().getBytes()).toString().substring(0, 80).replaceAll("\\s+", " ")), + DOUBLE, + null); + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testEmptySerialization() + { + functionAssertions.assertFunction(format("value_at_quantile(CAST(X'%s' AS tdigest(double)), 0.5)", + new SqlVarbinary(new byte[0])), + DOUBLE, + null); + } + + @Test + public void testMergeTwoNormalDistributionsGetQuantile() + { + TDigest tDigest1 = createTDigest(STANDARD_COMPRESSION_FACTOR); + TDigest tDigest2 = createTDigest(STANDARD_COMPRESSION_FACTOR); + List list = new ArrayList<>(); + NormalDistribution normal = new NormalDistribution(0, 50); + + for (int i = 0; i < NUMBER_OF_ENTRIES / 2; i++) { + double value1 = normal.sample(); + double value2 = normal.sample(); + tDigest1.add(value1); + tDigest2.add(value2); + list.add(value1); + list.add(value2); + } + + tDigest1.merge(tDigest2); + sort(list); + + for (int i = 0; i < quantiles.length; i++) { + assertValueWithinBound(quantiles[i], STANDARD_ERROR, list, tDigest1); + } + } + + @Test + public void testGetQuantileAtValueOutsideRange() + { + TDigest tDigest = createTDigest(STANDARD_COMPRESSION_FACTOR); + + for (int i = 0; i < NUMBER_OF_ENTRIES; i++) { + double value = Math.random() * NUMBER_OF_ENTRIES; + tDigest.add(value); + } + + functionAssertions.assertFunction( + format("quantile_at_value(CAST(X'%s' AS tdigest(%s)), %s) = 1", + new SqlVarbinary(tDigest.serialize().getBytes()).toString().replaceAll("\\s+", " "), + DOUBLE, + 1_000_000_000d), + BOOLEAN, + true); + + functionAssertions.assertFunction( + format("quantile_at_value(CAST(X'%s' AS tdigest(%s)), %s) = 0", + new SqlVarbinary(tDigest.serialize().getBytes()).toString().replaceAll("\\s+", " "), + DOUBLE, + -500d), + BOOLEAN, + true); + } + + @Test + public void testNormalDistributionHighVarianceValuesArray() + { + TDigest tDigest = createTDigest(STANDARD_COMPRESSION_FACTOR); + NormalDistribution normal = new NormalDistribution(0, 1); + List list = new ArrayList<>(); + + for (int i = 0; i < NUMBER_OF_ENTRIES; i++) { + double value = normal.sample(); + tDigest.add(value); + list.add(value); + } + + sort(list); + + double[] values = new double[quantiles.length]; + + for (int i = 0; i < quantiles.length; i++) { + values[i] = list.get((int) (quantiles[i] * NUMBER_OF_ENTRIES)); + } + + assertBlockValues(values, STANDARD_ERROR, tDigest); + } + + @Test + public void testAddElementsInOrderQuantileArray() + { + TDigest tDigest = createTDigest(STANDARD_COMPRESSION_FACTOR); + List list = new ArrayList<>(); + + for (int i = 0; i < NUMBER_OF_ENTRIES; i++) { + tDigest.add(i); + list.add((double) i); + } + + sort(list); + + assertBlockQuantiles(quantiles, STANDARD_ERROR, list, tDigest); + } + + @Test + public void testNormalDistributionHighVarianceQuantileArray() + { + TDigest tDigest = createTDigest(STANDARD_COMPRESSION_FACTOR); + List list = new ArrayList(); + NormalDistribution normal = new NormalDistribution(0, 1); + + for (int i = 0; i < NUMBER_OF_ENTRIES; i++) { + double value = normal.sample(); + tDigest.add(value); + list.add(value); + } + + sort(list); + + assertBlockQuantiles(quantiles, STANDARD_ERROR, list, tDigest); + } + + @Test + public void testAddElementsInOrder() + { + TDigest tDigest = createTDigest(STANDARD_COMPRESSION_FACTOR); + List list = new ArrayList<>(); + + for (int i = 0; i < NUMBER_OF_ENTRIES; i++) { + tDigest.add(i); + list.add(i); + } + + for (int i = 0; i < quantiles.length; i++) { + assertDiscreteQuantileWithinBound(quantiles[i], STANDARD_ERROR, list, tDigest); + } + } + + @Test + public void testMergeTwoDistributionsWithoutOverlap() + { + TDigest tDigest1 = createTDigest(STANDARD_COMPRESSION_FACTOR); + TDigest tDigest2 = createTDigest(STANDARD_COMPRESSION_FACTOR); + List list = new ArrayList(); + + for (int i = 0; i < NUMBER_OF_ENTRIES / 2; i++) { + tDigest1.add(i); + tDigest2.add(i + NUMBER_OF_ENTRIES / 2); + list.add(i); + list.add(i + NUMBER_OF_ENTRIES / 2); + } + + tDigest1.merge(tDigest2); + sort(list); + + for (int i = 0; i < quantiles.length; i++) { + assertDiscreteQuantileWithinBound(quantiles[i], STANDARD_ERROR, list, tDigest1); + } + } + + @Test + public void testMergeTwoDistributionsWithOverlap() + { + TDigest tDigest1 = createTDigest(STANDARD_COMPRESSION_FACTOR); + TDigest tDigest2 = createTDigest(STANDARD_COMPRESSION_FACTOR); + List list = new ArrayList(); + + for (int i = 0; i < NUMBER_OF_ENTRIES / 2; i++) { + tDigest1.add(i); + tDigest2.add(i); + list.add(i); + list.add(i); + } + + tDigest2.merge(tDigest1); + sort(list); + + for (int i = 0; i < quantiles.length; i++) { + assertDiscreteQuantileWithinBound(quantiles[i], STANDARD_ERROR, list, tDigest2); + } + } + + @Test + public void testAddElementsRandomized() + { + TDigest tDigest = createTDigest(STANDARD_COMPRESSION_FACTOR); + List list = new ArrayList(); + + for (int i = 0; i < NUMBER_OF_ENTRIES; i++) { + double value = Math.random() * NUMBER_OF_ENTRIES; + tDigest.add(value); + list.add(value); + } + + sort(list); + + for (int i = 0; i < quantiles.length; i++) { + assertContinuousQuantileWithinBound(quantiles[i], STANDARD_ERROR, list, tDigest); + } + } + + @Test + public void testNormalDistributionLowVariance() + { + TDigest tDigest = createTDigest(STANDARD_COMPRESSION_FACTOR); + List list = new ArrayList(); + NormalDistribution normal = new NormalDistribution(1000, 1); + + for (int i = 0; i < NUMBER_OF_ENTRIES; i++) { + double value = normal.sample(); + tDigest.add(value); + list.add(value); + } + + sort(list); + + for (int i = 0; i < quantiles.length; i++) { + assertContinuousQuantileWithinBound(quantiles[i], STANDARD_ERROR, list, tDigest); + } + } + + @Test + public void testNormalDistributionHighVariance() + { + TDigest tDigest = createTDigest(STANDARD_COMPRESSION_FACTOR); + List list = new ArrayList(); + NormalDistribution normal = new NormalDistribution(0, 1); + + for (int i = 0; i < NUMBER_OF_ENTRIES; i++) { + double value = normal.sample(); + tDigest.add(value); + list.add(value); + } + + sort(list); + + for (int i = 0; i < quantiles.length; i++) { + assertContinuousQuantileWithinBound(quantiles[i], STANDARD_ERROR, list, tDigest); + } + } + + @Test + public void testMergeTwoNormalDistributions() + { + TDigest tDigest1 = createTDigest(STANDARD_COMPRESSION_FACTOR); + TDigest tDigest2 = createTDigest(STANDARD_COMPRESSION_FACTOR); + List list = new ArrayList<>(); + NormalDistribution normal = new NormalDistribution(0, 50); + + for (int i = 0; i < NUMBER_OF_ENTRIES / 2; i++) { + double value1 = normal.sample(); + double value2 = normal.sample(); + tDigest1.add(value1); + tDigest2.add(value2); + list.add(value1); + list.add(value2); + } + + tDigest1.merge(tDigest2); + sort(list); + + for (int i = 0; i < quantiles.length; i++) { + assertContinuousQuantileWithinBound(quantiles[i], STANDARD_ERROR, list, tDigest1); + } + } + + @Test + public void testMergeManySmallNormalDistributions() + { + TDigest tDigest = createTDigest(STANDARD_COMPRESSION_FACTOR); + List list = new ArrayList<>(); + NormalDistribution normal = new NormalDistribution(500, 20); + int digests = 100_000; + + for (int k = 0; k < digests; k++) { + TDigest current = createTDigest(STANDARD_COMPRESSION_FACTOR); + for (int i = 0; i < 10; i++) { + double value = normal.sample(); + current.add(value); + list.add(value); + } + tDigest.merge(current); + } + + sort(list); + + for (int i = 0; i < quantiles.length; i++) { + assertContinuousQuantileWithinBound(quantiles[i], STANDARD_ERROR, list, tDigest); + } + } + + @Test + public void testMergeManyLargeNormalDistributions() + { + TDigest tDigest = createTDigest(STANDARD_COMPRESSION_FACTOR); + List list = new ArrayList<>(); + NormalDistribution normal = new NormalDistribution(500, 20); + int digests = 1000; + + for (int k = 0; k < digests; k++) { + TDigest current = createTDigest(STANDARD_COMPRESSION_FACTOR); + for (int i = 0; i < NUMBER_OF_ENTRIES / digests; i++) { + double value = normal.sample(); + current.add(value); + list.add(value); + } + tDigest.merge(current); + } + + sort(list); + + for (int i = 0; i < quantiles.length; i++) { + assertContinuousQuantileWithinBound(quantiles[i], STANDARD_ERROR, list, tDigest); + } + } + + // disabled because test takes almost 10s + @Test(enabled = false) + public void testBinomialDistribution() + { + int trials = 10; + for (int k = 1; k < trials; k++) { + TDigest tDigest = createTDigest(STANDARD_COMPRESSION_FACTOR); + BinomialDistribution binomial = new BinomialDistribution(trials, k * 0.1); + List list = new ArrayList<>(); + + for (int i = 0; i < NUMBER_OF_ENTRIES; i++) { + int sample = binomial.sample(); + tDigest.add(sample); + list.add(sample); + } + + Collections.sort(list); + + for (int i = 0; i < quantiles.length; i++) { + assertDiscreteQuantileWithinBound(quantiles[i], STANDARD_ERROR, list, tDigest); + } + } + } + + @Test(enabled = false) + public void testGeometricDistribution() + { + int trials = 10; + for (int k = 1; k < trials; k++) { + TDigest tDigest = createTDigest(STANDARD_COMPRESSION_FACTOR); + GeometricDistribution geometric = new GeometricDistribution(k * 0.1); + List list = new ArrayList(); + + for (int i = 0; i < NUMBER_OF_ENTRIES; i++) { + int sample = geometric.sample(); + tDigest.add(sample); + list.add(sample); + } + + Collections.sort(list); + + for (int i = 0; i < quantiles.length; i++) { + assertDiscreteQuantileWithinBound(quantiles[i], STANDARD_ERROR, list, tDigest); + } + } + } + + @Test(enabled = false) + public void testPoissonDistribution() + { + int trials = 10; + for (int k = 1; k < trials; k++) { + TDigest tDigest = createTDigest(STANDARD_COMPRESSION_FACTOR); + PoissonDistribution poisson = new PoissonDistribution(k * 0.1); + List list = new ArrayList(); + + for (int i = 0; i < NUMBER_OF_ENTRIES; i++) { + int sample = poisson.sample(); + tDigest.add(sample); + list.add(sample); + } + + Collections.sort(list); + + for (int i = 0; i < quantiles.length; i++) { + assertDiscreteQuantileWithinBound(quantiles[i], STANDARD_ERROR, list, tDigest); + } + } + } + + private void assertValueWithinBound(double quantile, double error, List list, TDigest tDigest) + { + functionAssertions.assertFunction( + format("quantile_at_value(CAST(X'%s' AS tdigest(%s)), %s) <= %s", + new SqlVarbinary(tDigest.serialize().getBytes()).toString().replaceAll("\\s+", " "), + DOUBLE, + list.get((int) (NUMBER_OF_ENTRIES * quantile)), + getUpperBoundQuantile(quantile, error)), + BOOLEAN, + true); + functionAssertions.assertFunction( + format("quantile_at_value(CAST(X'%s' AS tdigest(%s)), %s) >= %s", + new SqlVarbinary(tDigest.serialize().getBytes()).toString().replaceAll("\\s+", " "), + DOUBLE, + list.get((int) (NUMBER_OF_ENTRIES * quantile)), + getLowerBoundQuantile(quantile, error)), + BOOLEAN, + true); + } + + private void assertDiscreteQuantileWithinBound(double quantile, double error, List list, TDigest tDigest) + { + functionAssertions.assertFunction( + format("round(value_at_quantile(CAST(X'%s' AS tdigest(%s)), %s)) <= %s", + new SqlVarbinary(tDigest.serialize().getBytes()).toString().replaceAll("\\s+", " "), + DOUBLE, + quantile, + getUpperBoundValue(quantile, error, list)), + BOOLEAN, + true); + functionAssertions.assertFunction( + format("round(value_at_quantile(CAST(X'%s' AS tdigest(%s)), %s)) >= %s", + new SqlVarbinary(tDigest.serialize().getBytes()).toString().replaceAll("\\s+", " "), + DOUBLE, + quantile, + getLowerBoundValue(quantile, error, list)), + BOOLEAN, + true); + } + + private void assertContinuousQuantileWithinBound(double quantile, double error, List list, TDigest tDigest) + { + functionAssertions.assertFunction( + format("value_at_quantile(CAST(X'%s' AS tdigest(%s)), %s) <= %s", + new SqlVarbinary(tDigest.serialize().getBytes()).toString().replaceAll("\\s+", " "), + DOUBLE, + quantile, + getUpperBoundValue(quantile, error, list)), + BOOLEAN, + true); + functionAssertions.assertFunction( + format("value_at_quantile(CAST(X'%s' AS tdigest(%s)), %s) >= %s", + new SqlVarbinary(tDigest.serialize().getBytes()).toString().replaceAll("\\s+", " "), + DOUBLE, + quantile, + getLowerBoundValue(quantile, error, list)), + BOOLEAN, + true); + } + + private double getLowerBoundValue(double quantile, double error, List values) + { + return values.get((int) Math.max(NUMBER_OF_ENTRIES * (quantile - error), 0)).doubleValue(); + } + + private double getUpperBoundValue(double quantile, double error, List values) + { + return values.get((int) Math.min(NUMBER_OF_ENTRIES * (quantile + error), values.size() - 1)).doubleValue(); + } + + private double getLowerBoundQuantile(double quantile, double error) + { + return Math.max(0, quantile - error); + } + + private double getUpperBoundQuantile(double quantile, double error) + { + return Math.min(1, quantile + error); + } + + private void assertBlockQuantiles(double[] percentiles, double error, List rows, TDigest tDigest) + { + List boxedPercentiles = Arrays.stream(percentiles).sorted().boxed().collect(toImmutableList()); + List lowerBounds = boxedPercentiles.stream().map(percentile -> getLowerBoundValue(percentile, error, rows)).collect(toImmutableList()); + List upperBounds = boxedPercentiles.stream().map(percentile -> getUpperBoundValue(percentile, error, rows)).collect(toImmutableList()); + + // Ensure that the lower bound of each item in the distribution is not greater than the chosen quantiles + functionAssertions.assertFunction( + format( + "zip_with(values_at_quantiles(CAST(X'%s' AS tdigest(%s)), ARRAY[%s]), ARRAY[%s], (value, lowerbound) -> value >= lowerbound)", + new SqlVarbinary(tDigest.serialize().getBytes()).toString().replaceAll("\\s+", " "), + "double", + ARRAY_JOINER.join(boxedPercentiles), + ARRAY_JOINER.join(lowerBounds)), + METADATA.getType(parseTypeSignature("array(boolean)")), + Collections.nCopies(percentiles.length, true)); + + // Ensure that the upper bound of each item in the distribution is not less than the chosen quantiles + functionAssertions.assertFunction( + format( + "zip_with(values_at_quantiles(CAST(X'%s' AS tdigest(%s)), ARRAY[%s]), ARRAY[%s], (value, upperbound) -> value <= upperbound)", + new SqlVarbinary(tDigest.serialize().getBytes()).toString().replaceAll("\\s+", " "), + "double", + ARRAY_JOINER.join(boxedPercentiles), + ARRAY_JOINER.join(upperBounds)), + METADATA.getType(parseTypeSignature("array(boolean)")), + Collections.nCopies(percentiles.length, true)); + } + + private void assertBlockValues(double[] values, double error, TDigest tDigest) + { + List boxedValues = Arrays.stream(values).sorted().boxed().collect(toImmutableList()); + List boxedPercentiles = Arrays.stream(quantiles).sorted().boxed().collect(toImmutableList()); + List lowerBounds = boxedPercentiles.stream().map(percentile -> getLowerBoundQuantile(percentile, error)).collect(toImmutableList()); + List upperBounds = boxedPercentiles.stream().map(percentile -> getUpperBoundQuantile(percentile, error)).collect(toImmutableList()); + + // Ensure that the lower bound of each item in the distribution is not greater than the chosen quantiles + functionAssertions.assertFunction( + format( + "zip_with(quantiles_at_values(CAST(X'%s' AS tdigest(%s)), ARRAY[%s]), ARRAY[%s], (value, lowerbound) -> value >= lowerbound)", + new SqlVarbinary(tDigest.serialize().getBytes()).toString().replaceAll("\\s+", " "), + "double", + ARRAY_JOINER.join(boxedValues), + ARRAY_JOINER.join(lowerBounds)), + METADATA.getType(parseTypeSignature("array(boolean)")), + Collections.nCopies(values.length, true)); + + // Ensure that the upper bound of each item in the distribution is not less than the chosen quantiles + functionAssertions.assertFunction( + format( + "zip_with(quantiles_at_values(CAST(X'%s' AS tdigest(%s)), ARRAY[%s]), ARRAY[%s], (value, upperbound) -> value <= upperbound)", + new SqlVarbinary(tDigest.serialize().getBytes()).toString().replaceAll("\\s+", " "), + "double", + ARRAY_JOINER.join(boxedValues), + ARRAY_JOINER.join(upperBounds)), + METADATA.getType(parseTypeSignature("array(boolean)")), + Collections.nCopies(values.length, true)); + } +} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java index 637f561d016d6..524211fa2bfe1 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java @@ -127,7 +127,8 @@ public void testDefaults() .setIndexLoaderTimeout(new Duration(20, SECONDS)) .setOptimizedRepartitioningEnabled(false) .setListBuiltInFunctionsOnly(true) - .setPartitioningPrecisionStrategy(PartitioningPrecisionStrategy.AUTOMATIC)); + .setPartitioningPrecisionStrategy(PartitioningPrecisionStrategy.AUTOMATIC) + .setExperimentalFunctionsEnabled(false)); } @Test @@ -211,6 +212,7 @@ public void testExplicitPropertyMappings() .put("experimental.optimized-repartitioning", "true") .put("list-built-in-functions-only", "false") .put("partitioning-precision-strategy", "PREFER_EXACT_PARTITIONING") + .put("experimental-functions-enabled", "true") .build(); FeaturesConfig expected = new FeaturesConfig() @@ -290,7 +292,8 @@ public void testExplicitPropertyMappings() .setIndexLoaderTimeout(new Duration(10, SECONDS)) .setOptimizedRepartitioningEnabled(true) .setListBuiltInFunctionsOnly(false) - .setPartitioningPrecisionStrategy(PartitioningPrecisionStrategy.PREFER_EXACT_PARTITIONING); + .setPartitioningPrecisionStrategy(PartitioningPrecisionStrategy.PREFER_EXACT_PARTITIONING) + .setExperimentalFunctionsEnabled(true); assertFullMapping(properties, expected); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/gen/TestVarArgsToArrayAdapterGenerator.java b/presto-main/src/test/java/com/facebook/presto/sql/gen/TestVarArgsToArrayAdapterGenerator.java index 706a0a227f1e1..612772baddc3a 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/gen/TestVarArgsToArrayAdapterGenerator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/gen/TestVarArgsToArrayAdapterGenerator.java @@ -22,6 +22,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.TypeManager; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; @@ -35,6 +36,7 @@ import static com.facebook.presto.metadata.BuiltInFunctionNamespaceManager.DEFAULT_NAMESPACE; import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static com.facebook.presto.spi.type.IntegerType.INTEGER; import static com.facebook.presto.sql.gen.TestVarArgsToArrayAdapterGenerator.TestVarArgsSum.VAR_ARGS_SUM; import static com.facebook.presto.sql.gen.VarArgsToArrayAdapterGenerator.generateVarArgsToArrayAdapter; @@ -89,9 +91,9 @@ private TestVarArgsSum() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return PUBLIC; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/AggregationFunction.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/AggregationFunction.java index c11e6a0c953d6..fdda8a8693e43 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/AggregationFunction.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/AggregationFunction.java @@ -16,6 +16,7 @@ import java.lang.annotation.Retention; import java.lang.annotation.Target; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static java.lang.annotation.ElementType.METHOD; import static java.lang.annotation.ElementType.TYPE; import static java.lang.annotation.RetentionPolicy.RUNTIME; @@ -34,7 +35,7 @@ */ boolean isOrderSensitive() default false; - boolean hidden() default false; + SqlFunctionVisibility visibility() default PUBLIC; String[] alias() default {}; } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/ScalarFunction.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/ScalarFunction.java index 43504a6c1925f..273f060278c38 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/ScalarFunction.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/ScalarFunction.java @@ -16,6 +16,7 @@ import java.lang.annotation.Retention; import java.lang.annotation.Target; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static java.lang.annotation.ElementType.METHOD; import static java.lang.annotation.ElementType.TYPE; import static java.lang.annotation.RetentionPolicy.RUNTIME; @@ -28,7 +29,7 @@ String[] alias() default {}; - boolean hidden() default false; + SqlFunctionVisibility visibility() default PUBLIC; boolean deterministic() default true; diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunction.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunction.java index a21c07a122623..a0ec06b138506 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunction.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunction.java @@ -17,7 +17,7 @@ public interface SqlFunction { Signature getSignature(); - boolean isHidden(); + SqlFunctionVisibility getVisibility(); boolean isDeterministic(); diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionVisibility.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionVisibility.java new file mode 100644 index 0000000000000..94f545091dcbf --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionVisibility.java @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.function; + +public enum SqlFunctionVisibility +{ + PUBLIC, + HIDDEN, + /** + * Marked to indicate that the function is experimental and may change in the future. They have the same visibility as HIDDEN by default, but + * can be toggled via session property or system property. + * + * @see SystemSessionProperties#isExperimentalFunctionsEnabled session property + * @see FeaturesConfig#isExperimentalFunctionsEnabled system property + */ + EXPERIMENTAL, +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedFunction.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedFunction.java index 68b3f9345f61e..7693c8afe1eae 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedFunction.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedFunction.java @@ -22,6 +22,7 @@ import java.util.Optional; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.collectingAndThen; @@ -85,9 +86,9 @@ public Signature getSignature() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return false; + return PUBLIC; } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/type/QuantileDigestParametricType.java b/presto-spi/src/main/java/com/facebook/presto/spi/type/QuantileDigestParametricType.java index c6012fafab415..26de566b60122 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/type/QuantileDigestParametricType.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/type/QuantileDigestParametricType.java @@ -14,11 +14,8 @@ package com.facebook.presto.spi.type; import java.util.List; - -import static java.lang.String.format; - public class QuantileDigestParametricType - implements ParametricType + extends StatisticalDigestParametricType { public static final QuantileDigestParametricType QDIGEST = new QuantileDigestParametricType(); @@ -29,22 +26,8 @@ public String getName() } @Override - public Type createType(TypeManager typeManager, List parameters) + public Type getType(List parameters) { - checkArgument(parameters.size() == 1, "QDIGEST type expects exactly one type as a parameter, got %s", parameters); - checkArgument( - parameters.get(0).getKind() == ParameterKind.TYPE, - "QDIGEST expects type as a parameter, got %s", - parameters); - // Validation check on the acceptable type (bigint, real, double) intentionally omitted - // because this is validated in each function and to allow for consistent error messaging return new QuantileDigestType(parameters.get(0).getType()); } - - private static void checkArgument(boolean argument, String format, Object... args) - { - if (!argument) { - throw new IllegalArgumentException(format(format, args)); - } - } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/type/QuantileDigestType.java b/presto-spi/src/main/java/com/facebook/presto/spi/type/QuantileDigestType.java index d532a73c60928..5c82643a2815d 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/type/QuantileDigestType.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/type/QuantileDigestType.java @@ -13,71 +13,13 @@ */ package com.facebook.presto.spi.type; -import com.facebook.presto.spi.ConnectorSession; -import com.facebook.presto.spi.block.Block; -import com.facebook.presto.spi.block.BlockBuilder; -import com.fasterxml.jackson.annotation.JsonCreator; -import io.airlift.slice.Slice; +import static com.facebook.presto.spi.type.StandardTypes.QDIGEST; -import java.util.List; - -import static java.util.Collections.singletonList; - -public class QuantileDigestType - extends AbstractVariableWidthType +class QuantileDigestType + extends StatisticalDigestType { - private final Type type; - - @JsonCreator - public QuantileDigestType(Type type) - { - super(new TypeSignature(StandardTypes.QDIGEST, TypeSignatureParameter.of(type.getTypeSignature())), Slice.class); - this.type = type; - } - - @Override - public void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - if (block.isNull(position)) { - blockBuilder.appendNull(); - } - else { - block.writeBytesTo(position, 0, block.getSliceLength(position), blockBuilder); - blockBuilder.closeEntry(); - } - } - - @Override - public Slice getSlice(Block block, int position) - { - return block.getSlice(position, 0, block.getSliceLength(position)); - } - - @Override - public void writeSlice(BlockBuilder blockBuilder, Slice value) - { - writeSlice(blockBuilder, value, 0, value.length()); - } - - @Override - public void writeSlice(BlockBuilder blockBuilder, Slice value, int offset, int length) - { - blockBuilder.writeBytes(value, offset, length).closeEntry(); - } - - @Override - public Object getObjectValue(ConnectorSession session, Block block, int position) - { - if (block.isNull(position)) { - return null; - } - - return new SqlVarbinary(block.getSlice(position, 0, block.getSliceLength(position)).getBytes()); - } - - @Override - public List getTypeParameters() + QuantileDigestType(Type type) { - return singletonList(type); + super(QDIGEST, type); } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/type/StandardTypes.java b/presto-spi/src/main/java/com/facebook/presto/spi/type/StandardTypes.java index 253d5bb2612af..daf44c57be814 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/type/StandardTypes.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/type/StandardTypes.java @@ -26,6 +26,7 @@ public final class StandardTypes public static final String DOUBLE = "double"; public static final String HYPER_LOG_LOG = "HyperLogLog"; public static final String QDIGEST = "qdigest"; + public static final String TDIGEST = "tdigest"; public static final String P4_HYPER_LOG_LOG = "P4HyperLogLog"; public static final String INTERVAL_DAY_TO_SECOND = "interval day to second"; public static final String INTERVAL_YEAR_TO_MONTH = "interval year to month"; diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/type/StatisticalDigestParametricType.java b/presto-spi/src/main/java/com/facebook/presto/spi/type/StatisticalDigestParametricType.java new file mode 100644 index 0000000000000..9036abb4b746a --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/type/StatisticalDigestParametricType.java @@ -0,0 +1,45 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.type; + +import java.util.List; + +import static java.lang.String.format; + +public abstract class StatisticalDigestParametricType + implements ParametricType +{ + @Override + public Type createType(TypeManager typeManager, List parameters) + { + checkArgument(parameters.size() == 1, "%s type expects exactly one type as a parameter, got %s", this.getName(), parameters); + checkArgument( + parameters.get(0).getKind() == ParameterKind.TYPE, + "%s expects type as a parameter, got %s", + this.getName(), + parameters); + // Validation check on the acceptable type (bigint, real, double) intentionally omitted + // because this is validated in each function and to allow for consistent error messaging + return getType(parameters); + } + + protected abstract Type getType(List parameters); + + protected static void checkArgument(boolean argument, String format, Object... args) + { + if (!argument) { + throw new IllegalArgumentException(format(format, args)); + } + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/type/StatisticalDigestType.java b/presto-spi/src/main/java/com/facebook/presto/spi/type/StatisticalDigestType.java new file mode 100644 index 0000000000000..d609fe42225bb --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/type/StatisticalDigestType.java @@ -0,0 +1,83 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.type; + +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.block.Block; +import com.facebook.presto.spi.block.BlockBuilder; +import com.fasterxml.jackson.annotation.JsonCreator; +import io.airlift.slice.Slice; + +import java.util.List; + +import static java.util.Collections.singletonList; + +public abstract class StatisticalDigestType + extends AbstractVariableWidthType +{ + private final Type type; + + @JsonCreator + public StatisticalDigestType(String name, Type type) + { + super(new TypeSignature(name, TypeSignatureParameter.of(type.getTypeSignature())), Slice.class); + this.type = type; + } + + @Override + public void appendTo(Block block, int position, BlockBuilder blockBuilder) + { + if (block.isNull(position)) { + blockBuilder.appendNull(); + } + else { + block.writeBytesTo(position, 0, block.getSliceLength(position), blockBuilder); + blockBuilder.closeEntry(); + } + } + + @Override + public Slice getSlice(Block block, int position) + { + return block.getSlice(position, 0, block.getSliceLength(position)); + } + + @Override + public void writeSlice(BlockBuilder blockBuilder, Slice value) + { + writeSlice(blockBuilder, value, 0, value.length()); + } + + @Override + public void writeSlice(BlockBuilder blockBuilder, Slice value, int offset, int length) + { + blockBuilder.writeBytes(value, offset, length).closeEntry(); + } + + @Override + public Object getObjectValue(ConnectorSession session, Block block, int position) + { + if (block.isNull(position)) { + return null; + } + + return new SqlVarbinary(block.getSlice(position, 0, block.getSliceLength(position)).getBytes()); + } + + @Override + public List getTypeParameters() + { + return singletonList(type); + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/type/TDigestParametricType.java b/presto-spi/src/main/java/com/facebook/presto/spi/type/TDigestParametricType.java new file mode 100644 index 0000000000000..d829f21e411e0 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/type/TDigestParametricType.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.type; + +import java.util.List; + +public class TDigestParametricType + extends StatisticalDigestParametricType +{ + public static final TDigestParametricType TDIGEST = new TDigestParametricType(); + + @Override + public String getName() + { + return StandardTypes.TDIGEST; + } + + @Override + public Type getType(List parameters) + { + return new TDigestType(parameters.get(0).getType()); + } +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/type/TDigestType.java b/presto-spi/src/main/java/com/facebook/presto/spi/type/TDigestType.java new file mode 100644 index 0000000000000..bf056df74a44c --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/type/TDigestType.java @@ -0,0 +1,25 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.type; + +import static com.facebook.presto.spi.type.StandardTypes.TDIGEST; + +class TDigestType + extends StatisticalDigestType +{ + TDigestType(Type type) + { + super(TDIGEST, type); + } +} diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestAggregations.java b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestAggregations.java index 9ad7a2181f5db..3b1314f908c83 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestAggregations.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/AbstractTestAggregations.java @@ -13,16 +13,18 @@ */ package com.facebook.presto.tests; -import com.facebook.presto.Session; import com.facebook.presto.testing.MaterializedResult; import com.facebook.presto.testing.MaterializedRow; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.util.List; +import static com.facebook.presto.Session.builder; import static com.facebook.presto.SessionTestUtils.TEST_SESSION; import static com.facebook.presto.testing.MaterializedResult.resultBuilder; import static com.facebook.presto.tests.QueryAssertions.assertEqualsIgnoreOrder; +import static java.lang.String.format; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -212,7 +214,7 @@ public void testCountDistinct() " GROUP BY orderkey " + "HAVING COUNT(DISTINCT partkey) != CARDINALITY(ARRAY_DISTINCT(ARRAY_AGG(partkey))))", "VALUES 0"); - assertQuery(Session.builder(TEST_SESSION) + assertQuery(builder(TEST_SESSION) .setSystemProperty("use_mark_distinct", "false") .build(), " SELECT COUNT(*) FROM (SELECT orderkey, COUNT(DISTINCT partkey) FROM lineitem " + @@ -1284,58 +1286,73 @@ public void testOrderedAggregations() } /** - * Comprehensive correctness testing is done in the TestQuantileDigestAggregationFunction + * Comprehensive correctness testing is done in the TestQuantileDigestAggregationFunction and TestTDigestAggregationFunction */ - @Test - public void testQuantileDigest() + @Test(dataProvider = "getType") + public void testStatisticalDigest(String type) { - assertQuery("SELECT value_at_quantile(qdigest_agg(orderkey), 0.5E0) > 0 FROM lineitem", "SELECT true"); - assertQuery("SELECT value_at_quantile(qdigest_agg(quantity), 0.5E0) > 0 FROM lineitem", "SELECT true"); - assertQuery("SELECT value_at_quantile(qdigest_agg(CAST(quantity AS REAL)), 0.5E0) > 0 FROM lineitem", "SELECT true"); - assertQuery("SELECT value_at_quantile(qdigest_agg(orderkey, 2), 0.5E0) > 0 FROM lineitem", "SELECT true"); - assertQuery("SELECT value_at_quantile(qdigest_agg(quantity, 3), 0.5E0) > 0 FROM lineitem", "SELECT true"); - assertQuery("SELECT value_at_quantile(qdigest_agg(CAST(quantity AS REAL), 4), 0.5E0) > 0 FROM lineitem", "SELECT true"); - assertQuery("SELECT value_at_quantile(qdigest_agg(orderkey, 2, 0.0001E0), 0.5E0) > 0 FROM lineitem", "SELECT true"); - assertQuery("SELECT value_at_quantile(qdigest_agg(quantity, 3, 0.0001E0), 0.5E0) > 0 FROM lineitem", "SELECT true"); - assertQuery("SELECT value_at_quantile(qdigest_agg(CAST(quantity AS REAL), 4, 0.0001E0), 0.5E0) > 0 FROM lineitem", "SELECT true"); + assertQuery(format("SELECT value_at_quantile(%s_agg(CAST(orderkey AS DOUBLE)), 0.5E0) > 0 FROM lineitem", type), "SELECT true"); + assertQuery(format("SELECT value_at_quantile(%s_agg(CAST(quantity AS DOUBLE)), 0.5E0) > 0 FROM lineitem", type), "SELECT true"); + assertQuery(format("SELECT value_at_quantile(%s_agg(CAST(quantity AS DOUBLE)), 0.5E0) > 0 FROM lineitem", type), "SELECT true"); + assertQuery(format("SELECT value_at_quantile(%s_agg(CAST(orderkey AS DOUBLE), 2), 0.5E0) > 0 FROM lineitem", type), "SELECT true"); + assertQuery(format("SELECT value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 3), 0.5E0) > 0 FROM lineitem", type), "SELECT true"); + assertQuery(format("SELECT value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 4), 0.5E0) > 0 FROM lineitem", type), "SELECT true"); + assertQuery(format("SELECT value_at_quantile(%s_agg(CAST(orderkey AS DOUBLE), 2, 0.0001E0), 0.5E0) > 0 FROM lineitem", type), "SELECT true"); + assertQuery(format("SELECT value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 3, 0.0001E0), 0.5E0) > 0 FROM lineitem", type), "SELECT true"); + assertQuery(format("SELECT value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 4, 0.0001E0), 0.5E0) > 0 FROM lineitem", type), "SELECT true"); } /** - * Comprehensive correctness testing is done in the TestQuantileDigestAggregationFunction + * Comprehensive correctness testing is done in the TestQuantileDigestAggregationFunction and TestTDigestAggregationFunction */ - @Test - public void testQuantileDigestGroupBy() + @Test(dataProvider = "getType") + public void testStatisticalDigestGroupBy(String type) { - assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(orderkey), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); - assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(quantity), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); - assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(CAST(quantity AS REAL)), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); - assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(orderkey, 2), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); - assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(quantity, 3), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); - assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(CAST(quantity AS REAL), 4), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); - assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(orderkey, 2, 0.0001E0), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); - assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(quantity, 3, 0.0001E0), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); - assertQuery("SELECT partkey, value_at_quantile(qdigest_agg(CAST(quantity AS REAL), 4, 0.0001E0), 0.5E0) > 0 FROM lineitem GROUP BY partkey", "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery(format("SELECT partkey, value_at_quantile(%s_agg(CAST(orderkey AS DOUBLE)), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery(format("SELECT partkey, value_at_quantile(%s_agg(CAST(quantity AS DOUBLE)), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery(format("SELECT partkey, value_at_quantile(%s_agg(CAST(quantity AS DOUBLE)), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery(format("SELECT partkey, value_at_quantile(%s_agg(CAST(orderkey AS DOUBLE), 2), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery(format("SELECT partkey, value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 3), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery(format("SELECT partkey, value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 4), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery(format("SELECT partkey, value_at_quantile(%s_agg(CAST(orderkey AS DOUBLE), 2, 0.0001E0), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery(format("SELECT partkey, value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 3, 0.0001E0), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + "SELECT partkey, true FROM lineitem GROUP BY partkey"); + assertQuery(format("SELECT partkey, value_at_quantile(%s_agg(CAST(quantity AS DOUBLE), 4, 0.0001E0), 0.5E0) > 0 FROM lineitem GROUP BY partkey", type), + "SELECT partkey, true FROM lineitem GROUP BY partkey"); } /** - * Comprehensive correctness testing is done in the TestMergeQuantileDigestFunction + * Comprehensive correctness testing is done in the TestMergeQuantileDigestFunction and TestMergeTDigestFunction */ - @Test - public void testQuantileDigestMerge() + @Test(dataProvider = "getType") + public void testStatisticalDigestMerge(String type) { - assertQuery("SELECT value_at_quantile(merge(qdigest), 0.5E0) > 0 FROM (SELECT partkey, qdigest_agg(orderkey) as qdigest FROM lineitem GROUP BY partkey)", "SELECT true"); + assertQuery(format("SELECT value_at_quantile(merge(%s), 0.5E0) > 0 FROM (SELECT partkey, %s_agg(CAST(orderkey AS DOUBLE)) as %s FROM lineitem GROUP BY partkey)", + type, + type, + type), + "SELECT true"); } /** - * Comprehensive correctness testing is done in the TestMergeQuantileDigestFunction + * Comprehensive correctness testing is done in the TestMergeQuantileDigestFunction and TestMergeTDigestFunction */ - @Test - public void testQuantileDigestMergeGroupBy() + @Test(dataProvider = "getType") + public void testStatisticalDigestMergeGroupBy(String type) { - assertQuery("" + - "SELECT partkey, value_at_quantile(merge(qdigest), 0.5E0) > 0 " + - "FROM (SELECT partkey, suppkey, qdigest_agg(orderkey) as qdigest FROM lineitem GROUP BY partkey, suppkey)" + + assertQuery(format("SELECT partkey, value_at_quantile(merge(%s), 0.5E0) > 0 " + + "FROM (SELECT partkey, suppkey, %s_agg(CAST(orderkey AS DOUBLE)) as %s FROM lineitem GROUP BY partkey, suppkey)" + "GROUP BY partkey", + type, + type, + type), "SELECT partkey, true FROM lineitem GROUP BY partkey"); } @@ -1351,4 +1368,10 @@ public void testGroupedRow() ")", "SELECT 15000, 15000"); } + + @DataProvider(name = "getType") + protected Object[][] getDigests() + { + return new Object[][] {{"tdigest"}, {"qdigest"}}; + } } diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/StatefulSleepingSum.java b/presto-tests/src/main/java/com/facebook/presto/tests/StatefulSleepingSum.java index 35abb77c33670..66f26874cbd11 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/StatefulSleepingSum.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/StatefulSleepingSum.java @@ -20,6 +20,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.QualifiedFunctionName; import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionVisibility; import com.facebook.presto.spi.type.TypeManager; import com.google.common.collect.ImmutableList; @@ -31,6 +32,7 @@ import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.Signature.typeVariable; +import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.util.Reflection.constructorMethodHandle; import static com.facebook.presto.util.Reflection.methodHandle; @@ -55,9 +57,9 @@ private StatefulSleepingSum() } @Override - public boolean isHidden() + public SqlFunctionVisibility getVisibility() { - return true; + return HIDDEN; } @Override