Add t-digest functions to Presto

presto-benchmark/src/test/java/com/facebook/presto/benchmark/BenchmarkDoubleStatisticalDigestAggregationFunctions.java

@@ -0,0 +1,108 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.benchmark;
+
+import com.facebook.presto.plugin.memory.MemoryConnectorFactory;
+import com.facebook.presto.testing.LocalQueryRunner;
+import com.facebook.presto.testing.MaterializedResult;
+import com.facebook.presto.tpch.TpchConnectorFactory;
+import com.google.common.collect.ImmutableMap;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.io.IOException;
+
+import static com.facebook.presto.testing.TestingSession.testSessionBuilder;
+import static java.lang.String.format;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.openjdk.jmh.annotations.Mode.AverageTime;
+
+@OutputTimeUnit(MILLISECONDS)
+@BenchmarkMode(AverageTime)
+@Fork(2)
+@Warmup(iterations = 10)
+@Measurement(iterations = 10)
+public class BenchmarkDoubleStatisticalDigestAggregationFunctions
+{
+    @State(Scope.Thread)
+    public static class Context
+    {
+        @Param({"1", "2", "3"})
+        int scalingFactor;
+
+        private LocalQueryRunner queryRunner;
+
+        public LocalQueryRunner getQueryRunner()
+        {
+            return queryRunner;
+        }
+
+        @Setup
+        public void setUp()
+                throws IOException
+        {
+            queryRunner = new LocalQueryRunner(testSessionBuilder()
+                    .setCatalog("memory")
+                    .setSchema("default")
+                    .build());
+            queryRunner.createCatalog("tpch", new TpchConnectorFactory(), ImmutableMap.of());
+            queryRunner.createCatalog("memory", new MemoryConnectorFactory(), ImmutableMap.of());
+
+            queryRunner.execute(format("CREATE TABLE memory.default.testingtable AS SELECT totalprice from tpch.sf%s.orders", scalingFactor));
+        }
+
+        @TearDown
+        public void tearDown()
+        {
+            queryRunner.close();
+        }
+    }
+
+    @Benchmark
+    public MaterializedResult benchmarkTDigestAggregation(Context context)
+    {
+        return context.getQueryRunner()
+                .execute("SELECT tdigest_agg(totalprice) FROM memory.default.testingtable");
+    }
+
+    @Benchmark
+    public MaterializedResult benchmarkQuantileDigestAggregation(Context context)
+    {
+        return context.getQueryRunner()
+                .execute("SELECT qdigest_agg(totalprice) FROM memory.default.testingtable");
+    }
+
+    public static void main(String[] args)
+            throws Exception
+    {
+        Options options = new OptionsBuilder()
+                .include(".*" + BenchmarkDoubleStatisticalDigestAggregationFunctions.class.getSimpleName() + ".*")
+                .build();
+
+        new Runner(options)
+                .run();
+    }
+}

presto-geospatial/src/main/java/com/facebook/presto/plugin/geospatial/SpatialPartitioningInternalAggregateFunction.java

@@ -35,11 +35,12 @@
 import static com.facebook.presto.plugin.geospatial.GeometryType.GEOMETRY_TYPE_NAME;
 import static com.facebook.presto.plugin.geospatial.SpatialPartitioningAggregateFunction.NAME;
 import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT;
+import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN;
 import static com.facebook.presto.spi.type.StandardTypes.INTEGER;
 import static com.facebook.presto.spi.type.VarcharType.VARCHAR;
 import static java.lang.Math.toIntExact;
 
-@AggregationFunction(value = NAME, decomposable = false, hidden = true)
+@AggregationFunction(value = NAME, decomposable = false, visibility = HIDDEN)
 public class SpatialPartitioningInternalAggregateFunction
 {
     private static final int MAX_SAMPLE_COUNT = 1_000_000;

presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java

@@ -141,6 +141,7 @@ public final class SystemSessionProperties
     public static final String AGGREGATION_PARTITIONING_MERGING_STRATEGY = "aggregation_partitioning_merging_strategy";
     public static final String LIST_BUILT_IN_FUNCTIONS_ONLY = "list_built_in_functions_only";
     public static final String PARTITIONING_PRECISION_STRATEGY = "partitioning_precision_strategy";
+    public static final String EXPERIMENTAL_FUNCTIONS_ENABLED = "experimental_functions_enabled";
 
     private final List<PropertyMetadata<?>> sessionProperties;
 
@@ -704,7 +705,12 @@ public SystemSessionProperties(
                         featuresConfig.getPartitioningPrecisionStrategy(),
                         false,
                         value -> PartitioningPrecisionStrategy.valueOf(((String) value).toUpperCase()),
-                        PartitioningPrecisionStrategy::name));
+                        PartitioningPrecisionStrategy::name),
+                booleanProperty(
+                        EXPERIMENTAL_FUNCTIONS_ENABLED,
+                        "Enable listing of functions marked as experimental",
+                        featuresConfig.isExperimentalFunctionsEnabled(),
+                        false));
     }
 
     public List<PropertyMetadata<?>> getSessionProperties()
@@ -1195,4 +1201,9 @@ public static boolean isExactPartitioningPreferred(Session session)
         return session.getSystemProperty(PARTITIONING_PRECISION_STRATEGY, PartitioningPrecisionStrategy.class)
                 == PartitioningPrecisionStrategy.PREFER_EXACT_PARTITIONING;
     }
+
+    public static boolean isExperimentalFunctionsEnabled(Session session)
+    {
+        return session.getSystemProperty(EXPERIMENTAL_FUNCTIONS_ENABLED, Boolean.class);
+    }
 }

presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunctionNamespaceManager.java

@@ -51,6 +51,7 @@
 import com.facebook.presto.operator.aggregation.MaxDataSizeForStats;
 import com.facebook.presto.operator.aggregation.MergeHyperLogLogAggregation;
 import com.facebook.presto.operator.aggregation.MergeQuantileDigestFunction;
+import com.facebook.presto.operator.aggregation.MergeTDigestFunction;
 import com.facebook.presto.operator.aggregation.RealCorrelationAggregation;
 import com.facebook.presto.operator.aggregation.RealCovarianceAggregation;
 import com.facebook.presto.operator.aggregation.RealGeometricMeanAggregations;
@@ -135,6 +136,7 @@
 import com.facebook.presto.operator.scalar.SplitToMapFunction;
 import com.facebook.presto.operator.scalar.SplitToMultimapFunction;
 import com.facebook.presto.operator.scalar.StringFunctions;
+import com.facebook.presto.operator.scalar.TDigestFunctions;
 import com.facebook.presto.operator.scalar.TryFunction;
 import com.facebook.presto.operator.scalar.TypeOfFunction;
 import com.facebook.presto.operator.scalar.UrlFunctions;
@@ -168,6 +170,7 @@
 import com.facebook.presto.spi.function.ScalarFunctionImplementation;
 import com.facebook.presto.spi.function.Signature;
 import com.facebook.presto.spi.function.SqlFunction;
+import com.facebook.presto.spi.function.SqlFunctionVisibility;
 import com.facebook.presto.spi.function.SqlInvokedFunction;
 import com.facebook.presto.spi.type.Type;
 import com.facebook.presto.spi.type.TypeManager;
@@ -192,6 +195,7 @@
 import com.facebook.presto.type.QuantileDigestOperators;
 import com.facebook.presto.type.RealOperators;
 import com.facebook.presto.type.SmallintOperators;
+import com.facebook.presto.type.TDigestOperators;
 import com.facebook.presto.type.TimeOperators;
 import com.facebook.presto.type.TimeWithTimeZoneOperators;
 import com.facebook.presto.type.TimestampOperators;
@@ -240,6 +244,9 @@
 import static com.facebook.presto.operator.aggregation.QuantileDigestAggregationFunction.QDIGEST_AGG_WITH_WEIGHT_AND_ERROR;
 import static com.facebook.presto.operator.aggregation.RealAverageAggregation.REAL_AVERAGE_AGGREGATION;
 import static com.facebook.presto.operator.aggregation.ReduceAggregationFunction.REDUCE_AGG;
+import static com.facebook.presto.operator.aggregation.TDigestAggregationFunction.TDIGEST_AGG;
+import static com.facebook.presto.operator.aggregation.TDigestAggregationFunction.TDIGEST_AGG_WITH_WEIGHT;
+import static com.facebook.presto.operator.aggregation.TDigestAggregationFunction.TDIGEST_AGG_WITH_WEIGHT_AND_COMPRESSION;
 import static com.facebook.presto.operator.aggregation.minmaxby.MaxByAggregationFunction.MAX_BY;
 import static com.facebook.presto.operator.aggregation.minmaxby.MaxByNAggregationFunction.MAX_BY_N_AGGREGATION;
 import static com.facebook.presto.operator.aggregation.minmaxby.MinByAggregationFunction.MIN_BY;
@@ -305,6 +312,7 @@
 import static com.facebook.presto.spi.function.FunctionKind.SCALAR;
 import static com.facebook.presto.spi.function.FunctionKind.WINDOW;
 import static com.facebook.presto.spi.function.OperatorType.tryGetOperatorType;
+import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN;
 import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature;
 import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypeSignatures;
 import static com.facebook.presto.sql.planner.LiteralEncoder.MAGIC_LITERAL_FUNCTION_PREFIX;
@@ -662,7 +670,11 @@ public BuiltInFunctionNamespaceManager(
                 .aggregate(BuildSetDigestAggregation.class)
                 .scalars(SetDigestFunctions.class)
                 .scalars(SetDigestOperators.class)
-                .scalars(WilsonInterval.class);
+                .scalars(WilsonInterval.class)
+                .scalars(TDigestOperators.class)
+                .scalars(TDigestFunctions.class)
+                .functions(TDIGEST_AGG, TDIGEST_AGG_WITH_WEIGHT, TDIGEST_AGG_WITH_WEIGHT_AND_COMPRESSION)
+                .function(MergeTDigestFunction.MERGE);
 
         switch (featuresConfig.getRegexLibrary()) {
             case JONI:
@@ -974,9 +986,9 @@ private static class MagicLiteralFunction
         }
 
         @Override
-        public boolean isHidden()
+        public final SqlFunctionVisibility getVisibility()
         {
-            return true;
+            return HIDDEN;
         }
 
         @Override

presto-main/src/main/java/com/facebook/presto/metadata/FunctionManager.java

@@ -14,6 +14,7 @@
 package com.facebook.presto.metadata;
 
 import com.facebook.presto.Session;
+import com.facebook.presto.SystemSessionProperties;
 import com.facebook.presto.operator.aggregation.InternalAggregationFunction;
 import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation;
 import com.facebook.presto.operator.window.WindowFunctionSupplier;
@@ -71,6 +72,8 @@
 import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_NOT_FOUND;
 import static com.facebook.presto.spi.StandardErrorCode.GENERIC_USER_ERROR;
 import static com.facebook.presto.spi.function.FunctionKind.SCALAR;
+import static com.facebook.presto.spi.function.SqlFunctionVisibility.EXPERIMENTAL;
+import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC;
 import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature;
 import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypeSignatures;
 import static com.facebook.presto.sql.planner.LiteralEncoder.MAGIC_LITERAL_FUNCTION_PREFIX;
@@ -179,7 +182,8 @@ public List<SqlFunction> listFunctions(Session session)
 
         return managers.stream()
                 .flatMap(manager -> manager.listFunctions().stream())
-                .filter(function -> !function.isHidden())
+                .filter(function -> function.getVisibility() == PUBLIC ||
+                        (function.getVisibility() == EXPERIMENTAL && SystemSessionProperties.isExperimentalFunctionsEnabled(session)))
                 .collect(toImmutableList());
     }
 

presto-main/src/main/java/com/facebook/presto/metadata/PolymorphicScalarFunction.java

@@ -22,6 +22,7 @@
 import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ReturnPlaceConvention;
 import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ScalarImplementationChoice;
 import com.facebook.presto.spi.function.Signature;
+import com.facebook.presto.spi.function.SqlFunctionVisibility;
 import com.facebook.presto.spi.type.Type;
 import com.facebook.presto.spi.type.TypeManager;
 import com.facebook.presto.util.Reflection;
@@ -45,32 +46,32 @@ class PolymorphicScalarFunction
         extends SqlScalarFunction
 {
     private final String description;
-    private final boolean hidden;
+    private final SqlFunctionVisibility visibility;
     private final boolean deterministic;
     private final boolean calledOnNullInput;
     private final List<PolymorphicScalarFunctionChoice> choices;
 
     PolymorphicScalarFunction(
             Signature signature,
             String description,
-            boolean hidden,
+            SqlFunctionVisibility visibility,
             boolean deterministic,
             boolean calledOnNullInput,
             List<PolymorphicScalarFunctionChoice> choices)
     {
         super(signature);
 
         this.description = description;
-        this.hidden = hidden;
+        this.visibility = visibility;
         this.deterministic = deterministic;
         this.calledOnNullInput = calledOnNullInput;
         this.choices = requireNonNull(choices, "choices is null");
     }
 
     @Override
-    public boolean isHidden()
+    public SqlFunctionVisibility getVisibility()
     {
-        return hidden;
+        return visibility;
     }
 
     @Override

presto-main/src/main/java/com/facebook/presto/metadata/PolymorphicScalarFunctionBuilder.java

@@ -18,6 +18,7 @@
 import com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ReturnPlaceConvention;
 import com.facebook.presto.spi.function.OperatorType;
 import com.facebook.presto.spi.function.Signature;
+import com.facebook.presto.spi.function.SqlFunctionVisibility;
 import com.facebook.presto.spi.type.Type;
 import com.facebook.presto.spi.type.TypeManager;
 import com.google.common.collect.ImmutableList;
@@ -32,6 +33,8 @@
 import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.ArgumentProperty.valueTypeArgumentProperty;
 import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.BLOCK_AND_POSITION;
 import static com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation.NullConvention.RETURN_NULL_ON_NULL;
+import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN;
+import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.ImmutableList.toImmutableList;
@@ -45,7 +48,7 @@ public final class PolymorphicScalarFunctionBuilder
     private final Optional<OperatorType> operatorType;
     private Signature signature;
     private String description;
-    private Optional<Boolean> hidden = Optional.empty();
+    private Optional<SqlFunctionVisibility> visibility = Optional.empty();
     private Boolean deterministic;
     private Boolean calledOnNullInput;
     private final List<PolymorphicScalarFunctionChoice> choices = new ArrayList<>();
@@ -65,7 +68,7 @@ public PolymorphicScalarFunctionBuilder(Class<?> clazz, OperatorType operatorTyp
     public PolymorphicScalarFunctionBuilder signature(Signature signature)
     {
         this.signature = requireNonNull(signature, "signature is null");
-        this.hidden = Optional.of(hidden.orElse(isOperator(signature)));
+        this.visibility = Optional.of(visibility.orElse(isOperator(signature) ? HIDDEN : PUBLIC));
         return this;
     }
 
@@ -75,9 +78,9 @@ public PolymorphicScalarFunctionBuilder description(String description)
         return this;
     }
 
-    public PolymorphicScalarFunctionBuilder hidden(boolean hidden)
+    public PolymorphicScalarFunctionBuilder visibility(SqlFunctionVisibility visibility)
     {
-        this.hidden = Optional.of(hidden);
+        this.visibility = Optional.of(visibility);
         return this;
     }
 
@@ -109,7 +112,7 @@ public SqlScalarFunction build()
         return new PolymorphicScalarFunction(
                 signature,
                 description,
-                hidden.orElse(false),
+                visibility.orElse(PUBLIC),
                 deterministic,
                 operatorType.map(OperatorType::isCalledOnNullInput).orElse(calledOnNullInput),
                 choices);