Add noisy_approx_distinct_sfm and related functions

presto-main/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java

@@ -93,7 +93,14 @@
 import com.facebook.presto.operator.aggregation.histogram.Histogram;
 import com.facebook.presto.operator.aggregation.multimapagg.AlternativeMultimapAggregationFunction;
 import com.facebook.presto.operator.aggregation.multimapagg.MultimapAggregationFunction;
+import com.facebook.presto.operator.aggregation.noisyaggregation.NoisyApproximateDistinctCountSfmAggregation;
+import com.facebook.presto.operator.aggregation.noisyaggregation.NoisyApproximateDistinctCountSfmAggregationDefaultBucketsPrecision;
+import com.facebook.presto.operator.aggregation.noisyaggregation.NoisyApproximateDistinctCountSfmAggregationDefaultPrecision;
+import com.facebook.presto.operator.aggregation.noisyaggregation.NoisyApproximateSetSfmAggregation;
+import com.facebook.presto.operator.aggregation.noisyaggregation.NoisyApproximateSetSfmAggregationDefaultBucketsPrecision;
+import com.facebook.presto.operator.aggregation.noisyaggregation.NoisyApproximateSetSfmAggregationDefaultPrecision;
 import com.facebook.presto.operator.aggregation.noisyaggregation.NoisyCountIfGaussianAggregation;
+import com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchMergeAggregation;
 import com.facebook.presto.operator.scalar.ArrayAllMatchFunction;
 import com.facebook.presto.operator.scalar.ArrayAnyMatchFunction;
 import com.facebook.presto.operator.scalar.ArrayCardinalityFunction;
@@ -173,6 +180,7 @@
 import com.facebook.presto.operator.scalar.RepeatFunction;
 import com.facebook.presto.operator.scalar.SequenceFunction;
 import com.facebook.presto.operator.scalar.SessionFunctions;
+import com.facebook.presto.operator.scalar.SfmSketchFunctions;
 import com.facebook.presto.operator.scalar.SplitToMapFunction;
 import com.facebook.presto.operator.scalar.SplitToMultimapFunction;
 import com.facebook.presto.operator.scalar.StringFunctions;
@@ -240,6 +248,7 @@
 import com.facebook.presto.type.MapParametricType;
 import com.facebook.presto.type.QuantileDigestOperators;
 import com.facebook.presto.type.RealOperators;
+import com.facebook.presto.type.SfmSketchOperators;
 import com.facebook.presto.type.SmallintOperators;
 import com.facebook.presto.type.TDigestOperators;
 import com.facebook.presto.type.TimeOperators;
@@ -489,6 +498,7 @@
 import static com.facebook.presto.type.MapParametricType.MAP;
 import static com.facebook.presto.type.Re2JRegexpType.RE2J_REGEXP;
 import static com.facebook.presto.type.RowParametricType.ROW;
+import static com.facebook.presto.type.SfmSketchType.SFM_SKETCH;
 import static com.facebook.presto.type.TypeUtils.resolveTypes;
 import static com.facebook.presto.type.khyperloglog.KHyperLogLogType.K_HYPER_LOG_LOG;
 import static com.facebook.presto.type.setdigest.SetDigestType.SET_DIGEST;
@@ -611,6 +621,7 @@ private void registerBuiltInTypes()
         addType(SET_DIGEST);
         addType(K_HYPER_LOG_LOG);
         addType(P4_HYPER_LOG_LOG);
+        addType(SFM_SKETCH);
         addType(JONI_REGEXP);
         addType(RE2J_REGEXP);
         addType(LIKE_PATTERN);
@@ -654,6 +665,12 @@ private List<? extends SqlFunction> getBuildInFunctions(FeaturesConfig featuresC
                 .window(LeadFunction.class)
                 .aggregate(ApproximateCountDistinctAggregation.class)
                 .aggregate(DefaultApproximateCountDistinctAggregation.class)
+                .aggregate(NoisyApproximateSetSfmAggregation.class)
+                .aggregate(NoisyApproximateSetSfmAggregationDefaultBucketsPrecision.class)
+                .aggregate(NoisyApproximateSetSfmAggregationDefaultPrecision.class)
+                .aggregate(NoisyApproximateDistinctCountSfmAggregation.class)
+                .aggregate(NoisyApproximateDistinctCountSfmAggregationDefaultBucketsPrecision.class)
+                .aggregate(NoisyApproximateDistinctCountSfmAggregationDefaultPrecision.class)
                 .aggregate(SumDataSizeForStats.class)
                 .aggregate(MaxDataSizeForStats.class)
                 .aggregate(ConvexHullAggregation.class)
@@ -688,6 +705,7 @@ private List<? extends SqlFunction> getBuildInFunctions(FeaturesConfig featuresC
                 .aggregates(GeometricMeanAggregations.class)
                 .aggregates(RealGeometricMeanAggregations.class)
                 .aggregates(MergeHyperLogLogAggregation.class)
+                .aggregates(SfmSketchMergeAggregation.class)
                 .aggregates(ApproximateSetAggregation.class)
                 .functions(QDIGEST_AGG, QDIGEST_AGG_WITH_WEIGHT, QDIGEST_AGG_WITH_WEIGHT_AND_ERROR)
                 .function(MergeQuantileDigestFunction.MERGE)
@@ -741,6 +759,7 @@ private List<? extends SqlFunction> getBuildInFunctions(FeaturesConfig featuresC
                 .scalars(KdbTreeCasts.class)
                 .scalar(ColorOperators.ColorDistinctFromOperator.class)
                 .scalars(HyperLogLogFunctions.class)
+                .scalars(SfmSketchFunctions.class)
                 .scalars(QuantileDigestFunctions.class)
                 .scalars(UnknownOperators.class)
                 .scalar(UnknownOperators.UnknownDistinctFromOperator.class)
@@ -778,6 +797,7 @@ private List<? extends SqlFunction> getBuildInFunctions(FeaturesConfig featuresC
                 .scalar(TimestampWithTimeZoneOperators.TimestampWithTimeZoneDistinctFromOperator.class)
                 .scalars(DateTimeOperators.class)
                 .scalars(HyperLogLogOperators.class)
+                .scalars(SfmSketchOperators.class)
                 .scalars(QuantileDigestOperators.class)
                 .scalars(IpAddressOperators.class)
                 .scalar(IpAddressOperators.IpAddressDistinctFromOperator.class)

presto-main/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyApproximateDistinctCountSfmAggregation.java

@@ -0,0 +1,93 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.operator.aggregation.noisyaggregation;
+
+import com.facebook.presto.common.block.BlockBuilder;
+import com.facebook.presto.common.type.StandardTypes;
+import com.facebook.presto.spi.function.AggregationFunction;
+import com.facebook.presto.spi.function.AggregationState;
+import com.facebook.presto.spi.function.CombineFunction;
+import com.facebook.presto.spi.function.InputFunction;
+import com.facebook.presto.spi.function.OperatorDependency;
+import com.facebook.presto.spi.function.OutputFunction;
+import com.facebook.presto.spi.function.SqlType;
+import com.facebook.presto.spi.function.TypeParameter;
+import io.airlift.slice.Slice;
+
+import java.lang.invoke.MethodHandle;
+
+import static com.facebook.presto.common.function.OperatorType.XX_HASH_64;
+import static com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchAggregationUtils.addHashToSketch;
+import static com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchAggregationUtils.hashDouble;
+import static com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchAggregationUtils.hashLong;
+import static com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchAggregationUtils.hashSlice;
+import static com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchAggregationUtils.mergeStates;
+import static com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchAggregationUtils.writeCardinality;
+
+@AggregationFunction(value = "noisy_approx_distinct_sfm")
+public final class NoisyApproximateDistinctCountSfmAggregation
+{
+    private NoisyApproximateDistinctCountSfmAggregation() {}
+
+    @InputFunction
+    @TypeParameter("T")
+    public static void input(
+            @OperatorDependency(operator = XX_HASH_64, argumentTypes = {"T"}) MethodHandle methodHandle,
+            @AggregationState SfmSketchState state,
+            @SqlType("T") long value,
+            @SqlType(StandardTypes.DOUBLE) double epsilon,
+            @SqlType(StandardTypes.BIGINT) long numberOfBuckets,
+            @SqlType(StandardTypes.BIGINT) long precision)
+    {
+        addHashToSketch(state, hashLong(methodHandle, value), epsilon, numberOfBuckets, precision);
+    }
+
+    @InputFunction
+    @TypeParameter("T")
+    public static void input(
+            @OperatorDependency(operator = XX_HASH_64, argumentTypes = {"T"}) MethodHandle methodHandle,
+            @AggregationState SfmSketchState state,
+            @SqlType("T") double value,
+            @SqlType(StandardTypes.DOUBLE) double epsilon,
+            @SqlType(StandardTypes.BIGINT) long numberOfBuckets,
+            @SqlType(StandardTypes.BIGINT) long precision)
+    {
+        addHashToSketch(state, hashDouble(methodHandle, value), epsilon, numberOfBuckets, precision);
+    }
+
+    @InputFunction
+    @TypeParameter("T")
+    public static void input(
+            @OperatorDependency(operator = XX_HASH_64, argumentTypes = {"T"}) MethodHandle methodHandle,
+            @AggregationState SfmSketchState state,
+            @SqlType("T") Slice value,
+            @SqlType(StandardTypes.DOUBLE) double epsilon,
+            @SqlType(StandardTypes.BIGINT) long numberOfBuckets,
+            @SqlType(StandardTypes.BIGINT) long precision)
+    {
+        addHashToSketch(state, hashSlice(methodHandle, value), epsilon, numberOfBuckets, precision);
+    }
+
+    @CombineFunction
+    public static void combineState(@AggregationState SfmSketchState state, @AggregationState SfmSketchState otherState)
+    {
+        mergeStates(state, otherState);
+    }
+
+    @OutputFunction(StandardTypes.BIGINT)
+    public static void evaluateFinal(@AggregationState SfmSketchState state, BlockBuilder out)
+    {
+        writeCardinality(state, out);
+    }
+}

presto-main/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyApproximateDistinctCountSfmAggregationDefaultBucketsPrecision.java

@@ -0,0 +1,89 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.operator.aggregation.noisyaggregation;
+
+import com.facebook.presto.common.block.BlockBuilder;
+import com.facebook.presto.common.type.StandardTypes;
+import com.facebook.presto.spi.function.AggregationFunction;
+import com.facebook.presto.spi.function.AggregationState;
+import com.facebook.presto.spi.function.CombineFunction;
+import com.facebook.presto.spi.function.InputFunction;
+import com.facebook.presto.spi.function.OperatorDependency;
+import com.facebook.presto.spi.function.OutputFunction;
+import com.facebook.presto.spi.function.SqlType;
+import com.facebook.presto.spi.function.TypeParameter;
+import io.airlift.slice.Slice;
+
+import java.lang.invoke.MethodHandle;
+
+import static com.facebook.presto.common.function.OperatorType.XX_HASH_64;
+import static com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchAggregationUtils.DEFAULT_BUCKET_COUNT;
+import static com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchAggregationUtils.DEFAULT_PRECISION;
+import static com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchAggregationUtils.addHashToSketch;
+import static com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchAggregationUtils.hashDouble;
+import static com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchAggregationUtils.hashLong;
+import static com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchAggregationUtils.hashSlice;
+import static com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchAggregationUtils.mergeStates;
+import static com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchAggregationUtils.writeCardinality;
+
+@AggregationFunction(value = "noisy_approx_distinct_sfm")
+public final class NoisyApproximateDistinctCountSfmAggregationDefaultBucketsPrecision
+{
+    private NoisyApproximateDistinctCountSfmAggregationDefaultBucketsPrecision() {}
+
+    @InputFunction
+    @TypeParameter("T")
+    public static void input(
+            @OperatorDependency(operator = XX_HASH_64, argumentTypes = {"T"}) MethodHandle methodHandle,
+            @AggregationState SfmSketchState state,
+            @SqlType("T") long value,
+            @SqlType(StandardTypes.DOUBLE) double epsilon)
+    {
+        addHashToSketch(state, hashLong(methodHandle, value), epsilon, DEFAULT_BUCKET_COUNT, DEFAULT_PRECISION);
+    }
+
+    @InputFunction
+    @TypeParameter("T")
+    public static void input(
+            @OperatorDependency(operator = XX_HASH_64, argumentTypes = {"T"}) MethodHandle methodHandle,
+            @AggregationState SfmSketchState state,
+            @SqlType("T") double value,
+            @SqlType(StandardTypes.DOUBLE) double epsilon)
+    {
+        addHashToSketch(state, hashDouble(methodHandle, value), epsilon, DEFAULT_BUCKET_COUNT, DEFAULT_PRECISION);
+    }
+
+    @InputFunction
+    @TypeParameter("T")
+    public static void input(
+            @OperatorDependency(operator = XX_HASH_64, argumentTypes = {"T"}) MethodHandle methodHandle,
+            @AggregationState SfmSketchState state,
+            @SqlType("T") Slice value,
+            @SqlType(StandardTypes.DOUBLE) double epsilon)
+    {
+        addHashToSketch(state, hashSlice(methodHandle, value), epsilon, DEFAULT_BUCKET_COUNT, DEFAULT_PRECISION);
+    }
+
+    @CombineFunction
+    public static void combineState(@AggregationState SfmSketchState state, @AggregationState SfmSketchState otherState)
+    {
+        mergeStates(state, otherState);
+    }
+
+    @OutputFunction(StandardTypes.BIGINT)
+    public static void evaluateFinal(@AggregationState SfmSketchState state, BlockBuilder out)
+    {
+        writeCardinality(state, out);
+    }
+}

presto-main/src/main/java/com/facebook/presto/operator/aggregation/noisyaggregation/NoisyApproximateDistinctCountSfmAggregationDefaultPrecision.java

@@ -0,0 +1,91 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.operator.aggregation.noisyaggregation;
+
+import com.facebook.presto.common.block.BlockBuilder;
+import com.facebook.presto.common.type.StandardTypes;
+import com.facebook.presto.spi.function.AggregationFunction;
+import com.facebook.presto.spi.function.AggregationState;
+import com.facebook.presto.spi.function.CombineFunction;
+import com.facebook.presto.spi.function.InputFunction;
+import com.facebook.presto.spi.function.OperatorDependency;
+import com.facebook.presto.spi.function.OutputFunction;
+import com.facebook.presto.spi.function.SqlType;
+import com.facebook.presto.spi.function.TypeParameter;
+import io.airlift.slice.Slice;
+
+import java.lang.invoke.MethodHandle;
+
+import static com.facebook.presto.common.function.OperatorType.XX_HASH_64;
+import static com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchAggregationUtils.DEFAULT_PRECISION;
+import static com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchAggregationUtils.addHashToSketch;
+import static com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchAggregationUtils.hashDouble;
+import static com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchAggregationUtils.hashLong;
+import static com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchAggregationUtils.hashSlice;
+import static com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchAggregationUtils.mergeStates;
+import static com.facebook.presto.operator.aggregation.noisyaggregation.SfmSketchAggregationUtils.writeCardinality;
+
+@AggregationFunction(value = "noisy_approx_distinct_sfm")
+public final class NoisyApproximateDistinctCountSfmAggregationDefaultPrecision
+{
+    private NoisyApproximateDistinctCountSfmAggregationDefaultPrecision() {}
+
+    @InputFunction
+    @TypeParameter("T")
+    public static void input(
+            @OperatorDependency(operator = XX_HASH_64, argumentTypes = {"T"}) MethodHandle methodHandle,
+            @AggregationState SfmSketchState state,
+            @SqlType("T") long value,
+            @SqlType(StandardTypes.DOUBLE) double epsilon,
+            @SqlType(StandardTypes.BIGINT) long numberOfBuckets)
+    {
+        addHashToSketch(state, hashLong(methodHandle, value), epsilon, numberOfBuckets, DEFAULT_PRECISION);
+    }
+
+    @InputFunction
+    @TypeParameter("T")
+    public static void input(
+            @OperatorDependency(operator = XX_HASH_64, argumentTypes = {"T"}) MethodHandle methodHandle,
+            @AggregationState SfmSketchState state,
+            @SqlType("T") double value,
+            @SqlType(StandardTypes.DOUBLE) double epsilon,
+            @SqlType(StandardTypes.BIGINT) long numberOfBuckets)
+    {
+        addHashToSketch(state, hashDouble(methodHandle, value), epsilon, numberOfBuckets, DEFAULT_PRECISION);
+    }
+
+    @InputFunction
+    @TypeParameter("T")
+    public static void input(
+            @OperatorDependency(operator = XX_HASH_64, argumentTypes = {"T"}) MethodHandle methodHandle,
+            @AggregationState SfmSketchState state,
+            @SqlType("T") Slice value,
+            @SqlType(StandardTypes.DOUBLE) double epsilon,
+            @SqlType(StandardTypes.BIGINT) long numberOfBuckets)
+    {
+        addHashToSketch(state, hashSlice(methodHandle, value), epsilon, numberOfBuckets, DEFAULT_PRECISION);
+    }
+
+    @CombineFunction
+    public static void combineState(@AggregationState SfmSketchState state, @AggregationState SfmSketchState otherState)
+    {
+        mergeStates(state, otherState);
+    }
+
+    @OutputFunction(StandardTypes.BIGINT)
+    public static void evaluateFinal(@AggregationState SfmSketchState state, BlockBuilder out)
+    {
+        writeCardinality(state, out);
+    }
+}