Extract all inlined sql invoked functions into a new plugin presto-sql-invoked-functions-plugin

pom.xml

@@ -221,6 +221,7 @@
         <module>presto-function-server</module>
         <module>presto-router-example-plugin-scheduler</module>
         <module>presto-plan-checker-router-plugin</module>
+        <module>presto-sql-invoked-functions-plugin</module>
         <module>presto-spark-classloader-spark${dep.pos.classloader.module-name.suffix}</module>
     </modules>
 

presto-hive/pom.xml

@@ -494,6 +494,12 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>com.facebook.presto</groupId>
+            <artifactId>presto-sql-invoked-functions-plugin</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>

presto-hive/src/test/java/com/facebook/presto/hive/TestDistributedQueriesSingleNode.java

@@ -13,6 +13,7 @@
  */
 package com.facebook.presto.hive;
 
+import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin;
 import com.facebook.presto.testing.QueryRunner;
 import com.facebook.presto.tests.AbstractTestDistributedQueries;
 import com.google.common.collect.ImmutableMap;
@@ -32,11 +33,13 @@ protected QueryRunner createQueryRunner()
     {
         ImmutableMap.Builder<String, String> coordinatorProperties = ImmutableMap.builder();
         coordinatorProperties.put("single-node-execution-enabled", "true");
-        return HiveQueryRunner.createQueryRunner(
+        QueryRunner queryRunner = HiveQueryRunner.createQueryRunner(
                 getTables(),
                 ImmutableMap.of(),
                 coordinatorProperties.build(),
                 Optional.empty());
+        queryRunner.installPlugin(new SqlInvokedFunctionsPlugin());
+        return queryRunner;
     }
 
     @Override

presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedNanQueries.java

@@ -14,6 +14,7 @@
 
 package com.facebook.presto.hive;
 
+import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin;
 import com.facebook.presto.testing.QueryRunner;
 import com.facebook.presto.tests.AbstractTestNanQueries;
 import com.google.common.collect.ImmutableList;
@@ -28,6 +29,9 @@ public class TestHiveDistributedNanQueries
     protected QueryRunner createQueryRunner()
             throws Exception
     {
-        return HiveQueryRunner.createQueryRunner(ImmutableList.of(), ImmutableMap.of("use-new-nan-definition", "true"), ImmutableMap.of(), Optional.empty());
+        QueryRunner queryRunner =
+                HiveQueryRunner.createQueryRunner(ImmutableList.of(), ImmutableMap.of("use-new-nan-definition", "true"), ImmutableMap.of(), Optional.empty());
+        queryRunner.installPlugin(new SqlInvokedFunctionsPlugin());
+        return queryRunner;
     }
 }

presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueries.java

@@ -15,6 +15,7 @@
 
 import com.facebook.presto.Session;
 import com.facebook.presto.hive.TestHiveEventListenerPlugin.TestingHiveEventListener;
+import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin;
 import com.facebook.presto.spi.QueryId;
 import com.facebook.presto.spi.eventlistener.EventListener;
 import com.facebook.presto.testing.MaterializedResult;
@@ -59,7 +60,9 @@ public class TestHiveDistributedQueries
     protected QueryRunner createQueryRunner()
             throws Exception
     {
-        return HiveQueryRunner.createQueryRunner(getTables());
+        QueryRunner queryRunner = HiveQueryRunner.createQueryRunner(getTables());
+        queryRunner.installPlugin(new SqlInvokedFunctionsPlugin());
+        return queryRunner;
     }
 
     @Override

presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithExchangeMaterialization.java

@@ -14,6 +14,7 @@
 package com.facebook.presto.hive;
 
 import com.facebook.presto.Session;
+import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin;
 import com.facebook.presto.testing.MaterializedResult;
 import com.facebook.presto.testing.QueryRunner;
 import com.facebook.presto.tests.AbstractTestDistributedQueries;
@@ -40,7 +41,9 @@ public class TestHiveDistributedQueriesWithExchangeMaterialization
     protected QueryRunner createQueryRunner()
             throws Exception
     {
-        return createMaterializingQueryRunner(getTables());
+        QueryRunner queryRunner = createMaterializingQueryRunner(getTables());
+        queryRunner.installPlugin(new SqlInvokedFunctionsPlugin());
+        return queryRunner;
     }
 
     @Test

presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithOptimizedRepartitioning.java

@@ -13,6 +13,7 @@
  */
 package com.facebook.presto.hive;
 
+import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin;
 import com.facebook.presto.testing.QueryRunner;
 import com.facebook.presto.tests.AbstractTestDistributedQueries;
 import com.google.common.collect.ImmutableMap;
@@ -30,13 +31,15 @@ public class TestHiveDistributedQueriesWithOptimizedRepartitioning
     protected QueryRunner createQueryRunner()
             throws Exception
     {
-        return HiveQueryRunner.createQueryRunner(
+        QueryRunner queryRunner = HiveQueryRunner.createQueryRunner(
                 getTables(),
                 ImmutableMap.of(
                         "experimental.optimized-repartitioning", "true",
                         // Use small SerializedPages to force flushing
                         "driver.max-page-partitioning-buffer-size", "10000B"),
                 Optional.empty());
+        queryRunner.installPlugin(new SqlInvokedFunctionsPlugin());
+        return queryRunner;
     }
 
     @Override

presto-hive/src/test/java/com/facebook/presto/hive/TestHiveDistributedQueriesWithThriftRpc.java

@@ -13,6 +13,7 @@
  */
 package com.facebook.presto.hive;
 
+import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin;
 import com.facebook.presto.testing.QueryRunner;
 import com.facebook.presto.tests.AbstractTestDistributedQueries;
 import com.google.common.collect.ImmutableMap;
@@ -30,13 +31,15 @@ public class TestHiveDistributedQueriesWithThriftRpc
     protected QueryRunner createQueryRunner()
             throws Exception
     {
-        return HiveQueryRunner.createQueryRunner(
+        QueryRunner queryRunner = HiveQueryRunner.createQueryRunner(
                 getTables(),
                 ImmutableMap.of(
                         "internal-communication.task-communication-protocol", "THRIFT",
                         "internal-communication.server-info-communication-protocol", "THRIFT"),
                 ImmutableMap.of(),
                 Optional.empty());
+        queryRunner.installPlugin(new SqlInvokedFunctionsPlugin());
+        return queryRunner;
     }
 
     @Override

presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownDistributedQueries.java

@@ -13,6 +13,7 @@
  */
 package com.facebook.presto.hive;
 
+import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin;
 import com.facebook.presto.testing.MaterializedResult;
 import com.facebook.presto.testing.QueryRunner;
 import com.facebook.presto.tests.AbstractTestDistributedQueries;
@@ -34,7 +35,7 @@ public class TestHivePushdownDistributedQueries
     protected QueryRunner createQueryRunner()
             throws Exception
     {
-        return HiveQueryRunner.createQueryRunner(
+        QueryRunner queryRunner = HiveQueryRunner.createQueryRunner(
                 getTables(),
                 ImmutableMap.of("experimental.pushdown-subfields-enabled", "true",
                         "experimental.pushdown-dereference-enabled", "true"),
@@ -44,6 +45,8 @@ protected QueryRunner createQueryRunner()
                         "hive.partial_aggregation_pushdown_enabled", "true",
                         "hive.partial_aggregation_pushdown_for_variable_length_datatypes_enabled", "true"),
                 Optional.empty());
+        queryRunner.installPlugin(new SqlInvokedFunctionsPlugin());
+        return queryRunner;
     }
 
     @Override

presto-hive/src/test/java/com/facebook/presto/hive/TestLambdaSubfieldPruning.java

@@ -14,6 +14,7 @@
 package com.facebook.presto.hive;
 
 import com.facebook.presto.Session;
+import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin;
 import com.facebook.presto.testing.QueryRunner;
 import com.facebook.presto.tests.AbstractTestQueryFramework;
 import com.facebook.presto.tests.DistributedQueryRunner;
@@ -126,6 +127,7 @@ private static DistributedQueryRunner createLineItemExTable(DistributedQueryRunn
 
                             "FROM lineitem  \n");
         }
+        queryRunner.installPlugin(new SqlInvokedFunctionsPlugin());
         return queryRunner;
     }
 

presto-hive/src/test/java/com/facebook/presto/hive/TestParquetDistributedQueries.java

@@ -14,6 +14,7 @@
 package com.facebook.presto.hive;
 
 import com.facebook.presto.Session;
+import com.facebook.presto.scalar.sql.SqlInvokedFunctionsPlugin;
 import com.facebook.presto.testing.MaterializedResult;
 import com.facebook.presto.testing.QueryRunner;
 import com.facebook.presto.tests.AbstractTestDistributedQueries;
@@ -46,14 +47,16 @@ protected QueryRunner createQueryRunner()
                 .put("hive.partial_aggregation_pushdown_enabled", "true")
                 .put("hive.partial_aggregation_pushdown_for_variable_length_datatypes_enabled", "true")
                 .build();
-        return HiveQueryRunner.createQueryRunner(
+        QueryRunner queryRunner = HiveQueryRunner.createQueryRunner(
                 getTables(),
                 ImmutableMap.of(
                         "experimental.pushdown-subfields-enabled", "true",
                         "experimental.pushdown-dereference-enabled", "true"),
                 "sql-standard",
                 parquetProperties,
                 Optional.empty());
+        queryRunner.installPlugin(new SqlInvokedFunctionsPlugin());
+        return queryRunner;
     }
 
     @Test

presto-i18n-functions/src/test/java/com/facebook/presto/i18n/functions/TestMyanmarFunctions.java

@@ -14,15 +14,23 @@
 package com.facebook.presto.i18n.functions;
 
 import com.facebook.presto.operator.scalar.AbstractTestFunctions;
+import com.facebook.presto.sql.analyzer.FeaturesConfig;
+import com.facebook.presto.sql.analyzer.FunctionsConfig;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
+import static com.facebook.presto.SessionTestUtils.TEST_SESSION;
 import static com.facebook.presto.common.type.VarcharType.VARCHAR;
 import static com.facebook.presto.metadata.FunctionExtractor.extractFunctions;
 
 public class TestMyanmarFunctions
         extends AbstractTestFunctions
 {
+    public TestMyanmarFunctions()
+    {
+        super(TEST_SESSION, new FeaturesConfig(), new FunctionsConfig(), false);
+    }
+
     @BeforeClass
     public void setUp()
     {

presto-main-base/pom.xml

@@ -461,9 +461,18 @@
             <groupId>io.netty</groupId>
             <artifactId>netty-transport</artifactId>
         </dependency>
+
         <dependency>
             <groupId>com.facebook.presto</groupId>
             <artifactId>presto-built-in-worker-function-tools</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.facebook.presto</groupId>
+            <artifactId>presto-sql-invoked-functions-plugin</artifactId>
+            <version>${project.version}</version>
             <scope>test</scope>
         </dependency>
     </dependencies>

presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInSpecialFunctionNamespaceManager.java

@@ -33,8 +33,6 @@
 import com.facebook.presto.spi.function.SqlFunction;
 import com.facebook.presto.spi.function.SqlInvokedFunction;
 import com.facebook.presto.spi.function.SqlInvokedScalarFunctionImplementation;
-import com.google.common.base.Supplier;
-import com.google.common.base.Suppliers;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
@@ -49,7 +47,6 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Throwables.throwIfInstanceOf;
 import static com.google.common.collect.ImmutableList.toImmutableList;
-import static java.util.Collections.emptyList;
 import static java.util.Objects.requireNonNull;
 import static java.util.concurrent.TimeUnit.HOURS;
 
@@ -58,8 +55,6 @@ public abstract class BuiltInSpecialFunctionNamespaceManager
 {
     protected volatile FunctionMap functions = new FunctionMap();
     private final FunctionAndTypeManager functionAndTypeManager;
-    protected final Supplier<FunctionMap> cachedFunctions =
-            Suppliers.memoize(this::createFunctionMap);
     private final LoadingCache<Signature, SpecializedFunctionKey> specializedFunctionKeyCache;
     private final LoadingCache<SpecializedFunctionKey, ScalarFunctionImplementation> specializedScalarCache;
 
@@ -85,11 +80,7 @@ public BuiltInSpecialFunctionNamespaceManager(FunctionAndTypeManager functionAnd
     @Override
     public Collection<SqlFunction> getFunctions(Optional<? extends FunctionNamespaceTransactionHandle> transactionHandle, QualifiedObjectName functionName)
     {
-        if (functions.list().isEmpty() ||
-                (!functionName.getCatalogSchemaName().equals(functionAndTypeManager.getDefaultNamespace()))) {
-            return emptyList();
-        }
-        return cachedFunctions.get().get(functionName);
+        return functions.get(functionName);
     }
 
     /**
@@ -98,7 +89,7 @@ public Collection<SqlFunction> getFunctions(Optional<? extends FunctionNamespace
     @Override
     public Collection<SqlFunction> listFunctions(Optional<String> likePattern, Optional<String> escape)
     {
-        return cachedFunctions.get().list();
+        return functions.list();
     }
 
     @Override

presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java

@@ -203,11 +203,6 @@
 import com.facebook.presto.operator.scalar.WilsonInterval;
 import com.facebook.presto.operator.scalar.WordStemFunction;
 import com.facebook.presto.operator.scalar.queryplan.JsonPrestoQueryPlanFunctions;
-import com.facebook.presto.operator.scalar.sql.ArraySqlFunctions;
-import com.facebook.presto.operator.scalar.sql.MapNormalizeFunction;
-import com.facebook.presto.operator.scalar.sql.MapSqlFunctions;
-import com.facebook.presto.operator.scalar.sql.SimpleSamplingPercent;
-import com.facebook.presto.operator.scalar.sql.StringSqlFunctions;
 import com.facebook.presto.operator.window.CumulativeDistributionFunction;
 import com.facebook.presto.operator.window.DenseRankFunction;
 import com.facebook.presto.operator.window.FirstValueFunction;
@@ -994,12 +989,6 @@ private List<? extends SqlFunction> getBuiltInFunctions(FunctionsConfig function
                 .aggregate(ThetaSketchAggregationFunction.class)
                 .scalars(ThetaSketchFunctions.class)
                 .function(MergeTDigestFunction.MERGE)
-                .sqlInvokedScalar(MapNormalizeFunction.class)
-                .sqlInvokedScalars(ArraySqlFunctions.class)
-                .sqlInvokedScalars(ArrayIntersectFunction.class)
-                .sqlInvokedScalars(MapSqlFunctions.class)
-                .sqlInvokedScalars(SimpleSamplingPercent.class)
-                .sqlInvokedScalars(StringSqlFunctions.class)
                 .scalar(DynamicFilterPlaceholderFunction.class)
                 .scalars(EnumCasts.class)
                 .scalars(LongEnumOperators.class)

presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionListBuilder.java

@@ -15,7 +15,6 @@
 
 import com.facebook.presto.operator.scalar.annotations.CodegenScalarFromAnnotationsParser;
 import com.facebook.presto.operator.scalar.annotations.ScalarFromAnnotationsParser;
-import com.facebook.presto.operator.scalar.annotations.SqlInvokedScalarFromAnnotationsParser;
 import com.facebook.presto.operator.window.WindowAnnotationsParser;
 import com.facebook.presto.spi.function.SqlFunction;
 import com.facebook.presto.spi.function.WindowFunction;
@@ -24,7 +23,6 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE;
 import static java.util.Objects.requireNonNull;
 
 public class FunctionListBuilder
@@ -61,18 +59,6 @@ public FunctionListBuilder scalars(Class<?> clazz)
         return this;
     }
 
-    public FunctionListBuilder sqlInvokedScalar(Class<?> clazz)
-    {
-        functions.addAll(SqlInvokedScalarFromAnnotationsParser.parseFunctionDefinition(clazz, JAVA_BUILTIN_NAMESPACE));
-        return this;
-    }
-
-    public FunctionListBuilder sqlInvokedScalars(Class<?> clazz)
-    {
-        functions.addAll(SqlInvokedScalarFromAnnotationsParser.parseFunctionDefinitions(clazz, JAVA_BUILTIN_NAMESPACE));
-        return this;
-    }
-
     public FunctionListBuilder codegenScalars(Class<?> clazz)
     {
         functions.addAll(CodegenScalarFromAnnotationsParser.parseFunctionDefinitions(clazz));

presto-main-base/src/main/java/com/facebook/presto/operator/scalar/ArrayIntersectFunction.java

@@ -19,8 +19,6 @@
 import com.facebook.presto.spi.function.Description;
 import com.facebook.presto.spi.function.OperatorDependency;
 import com.facebook.presto.spi.function.ScalarFunction;
-import com.facebook.presto.spi.function.SqlInvokedScalarFunction;
-import com.facebook.presto.spi.function.SqlParameter;
 import com.facebook.presto.spi.function.SqlType;
 import com.facebook.presto.spi.function.TypeParameter;
 
@@ -60,14 +58,4 @@ public static Block intersect(
 
         return typedSet.getBlock();
     }
-
-    @SqlInvokedScalarFunction(value = "array_intersect", deterministic = true, calledOnNullInput = false)
-    @Description("Intersects elements of all arrays in the given array")
-    @TypeParameter("T")
-    @SqlParameter(name = "input", type = "array<array<T>>")
-    @SqlType("array<T>")
-    public static String arrayIntersectArray()
-    {
-        return "RETURN reduce(input, IF((cardinality(input) = 0), ARRAY[], input[1]), (s, x) -> array_intersect(s, x), (s) -> s)";
-    }
 }