diff --git a/presto-docs/src/main/sphinx/develop/functions.rst b/presto-docs/src/main/sphinx/develop/functions.rst index 8afa657818aee..b30d3cfc71113 100644 --- a/presto-docs/src/main/sphinx/develop/functions.rst +++ b/presto-docs/src/main/sphinx/develop/functions.rst @@ -2,12 +2,15 @@ Functions ========= +Functions in Presto can be implemented at Plugin and the Connector level. +The following two sections describe how to implement them. + Plugin Implementation --------------------- The function framework is used to implement SQL functions. Presto includes a number of built-in functions. In order to implement new functions, you can -write a plugin that returns one more functions from ``getFunctions()``: +write a plugin that returns one or more functions from ``getFunctions()``: .. code-block:: java @@ -31,10 +34,44 @@ Note that the ``ImmutableSet`` class is a utility class from Guava. The ``getFunctions()`` method contains all of the classes for the functions that we will implement below in this tutorial. +Functions registered using this method are available in the default +namespace ``presto.default``. + For a full example in the codebase, see either the ``presto-ml`` module for machine learning functions or the ``presto-teradata-functions`` module for Teradata-compatible functions, both in the root of the Presto source. +Connector Functions Implementation +---------------------------------- + +To implement new functions at the connector level, in your +connector implementation, override the ``getSystemFunctions()`` method that returns one +or more functions: + +.. code-block:: java + + public class ExampleFunctionsConnector + implements Connector + { + @Override + public Set> getSystemFunctions() + { + return ImmutableSet.>builder() + .add(ExampleNullFunction.class) + .add(IsNullFunction.class) + .add(IsEqualOrNullFunction.class) + .add(ExampleStringFunction.class) + .add(ExampleAverageFunction.class) + .build(); + } + } + +Functions registered using this interface are available in the namespace +``.system`` where ```` is the catalog name used +in the Presto deployment for this connector type. + +At present, connector level functions do not support Window functions and Scalar operators. + Scalar Function Implementation ------------------------------ diff --git a/presto-main-base/src/main/java/com/facebook/presto/connector/ConnectorManager.java b/presto-main-base/src/main/java/com/facebook/presto/connector/ConnectorManager.java index ac86c7e7c3c19..dd84fbe4ff578 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/connector/ConnectorManager.java +++ b/presto-main-base/src/main/java/com/facebook/presto/connector/ConnectorManager.java @@ -15,6 +15,7 @@ import com.facebook.airlift.log.Logger; import com.facebook.airlift.node.NodeInfo; +import com.facebook.presto.common.CatalogSchemaName; import com.facebook.presto.common.block.BlockEncodingSerde; import com.facebook.presto.common.type.TypeManager; import com.facebook.presto.connector.informationSchema.InformationSchemaConnector; @@ -82,6 +83,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; +import static com.facebook.presto.metadata.FunctionExtractor.extractFunctions; import static com.facebook.presto.spi.ConnectorId.createInformationSchemaConnectorId; import static com.facebook.presto.spi.ConnectorId.createSystemTablesConnectorId; import static com.google.common.base.Preconditions.checkArgument; @@ -308,6 +310,10 @@ private synchronized void addConnectorInternal(MaterializedConnector connector) } connector.getConnectorCodecProvider().ifPresent(connectorCodecProvider -> connectorCodecManager.addConnectorCodecProvider(connectorId, connectorCodecProvider)); metadataManager.getProcedureRegistry().addProcedures(connectorId, connector.getProcedures()); + Set> systemFunctions = connector.getSystemFunctions(); + if (!systemFunctions.isEmpty()) { + metadataManager.registerConnectorFunctions(connectorId.getCatalogName(), extractFunctions(systemFunctions, new CatalogSchemaName(connectorId.getCatalogName(), "system"))); + } connector.getAccessControl() .ifPresent(accessControl -> accessControlManager.addCatalogAccessControl(connectorId, accessControl)); @@ -390,6 +396,8 @@ private static class MaterializedConnector private final ConnectorSplitManager splitManager; private final Set systemTables; private final Set procedures; + + private final Set> functions; private final ConnectorPageSourceProvider pageSourceProvider; private final Optional pageSinkProvider; private final Optional indexProvider; @@ -512,6 +520,10 @@ public MaterializedConnector(ConnectorId connectorId, Connector connector) List> analyzeProperties = connector.getAnalyzeProperties(); requireNonNull(analyzeProperties, "Connector %s returned a null analyze properties set"); this.analyzeProperties = ImmutableList.copyOf(analyzeProperties); + + Set> systemFunctions = connector.getSystemFunctions(); + requireNonNull(systemFunctions, "Connector %s returned a null system function set"); + this.functions = ImmutableSet.copyOf(systemFunctions); } public ConnectorId getConnectorId() @@ -539,6 +551,11 @@ public Set getProcedures() return procedures; } + public Set> getSystemFunctions() + { + return functions; + } + public ConnectorPageSourceProvider getPageSourceProvider() { return pageSourceProvider; diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java index eca3c9d6e1e79..f7565a43b47d2 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java @@ -552,6 +552,15 @@ public BuiltInTypeAndFunctionNamespaceManager( FunctionsConfig functionsConfig, Set types, FunctionAndTypeManager functionAndTypeManager) + { + this(blockEncodingSerde, functionsConfig, types, functionAndTypeManager, true); + } + public BuiltInTypeAndFunctionNamespaceManager( + BlockEncodingSerde blockEncodingSerde, + FunctionsConfig functionsConfig, + Set types, + FunctionAndTypeManager functionAndTypeManager, + boolean registerFunctions) { this.functionAndTypeManager = requireNonNull(functionAndTypeManager, "functionAndTypeManager is null"); this.magicLiteralFunction = new MagicLiteralFunction(blockEncodingSerde); @@ -605,7 +614,9 @@ public BuiltInTypeAndFunctionNamespaceManager( .expireAfterWrite(1, HOURS) .build(CacheLoader.from(this::instantiateParametricType)); - registerBuiltInFunctions(getBuiltInFunctions(functionsConfig)); + if (registerFunctions) { + registerBuiltInFunctions(getBuiltInFunctions(functionsConfig)); + } registerBuiltInTypes(functionsConfig); for (Type type : requireNonNull(types, "types is null")) { diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/DelegatingMetadataManager.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/DelegatingMetadataManager.java index d82aa9d6894ba..7302236671319 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/DelegatingMetadataManager.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/DelegatingMetadataManager.java @@ -86,6 +86,12 @@ public void registerBuiltInFunctions(List functionInfos) delegate.registerBuiltInFunctions(functionInfos); } + @Override + public void registerConnectorFunctions(String catalogName, List functionInfos) + { + delegate.registerConnectorFunctions(catalogName, functionInfos); + } + @Override public List listSchemaNames(Session session, String catalogName) { diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionAndTypeManager.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionAndTypeManager.java index 8c3dfe0f9b855..1755dd3abdc0e 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionAndTypeManager.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionAndTypeManager.java @@ -150,6 +150,8 @@ public class FunctionAndTypeManager private final AtomicReference servingTypeManager; private final AtomicReference>> servingTypeManagerParametricTypesSupplier; private final BuiltInPluginFunctionNamespaceManager builtInPluginFunctionNamespaceManager; + private final FunctionsConfig functionsConfig; + private final Set types; @Inject public FunctionAndTypeManager( @@ -162,6 +164,8 @@ public FunctionAndTypeManager( { this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); this.blockEncodingSerde = requireNonNull(blockEncodingSerde, "blockEncodingSerde is null"); + this.functionsConfig = requireNonNull(functionsConfig, "functionsConfig is null"); + this.types = requireNonNull(types, "types is null"); this.builtInTypeAndFunctionNamespaceManager = new BuiltInTypeAndFunctionNamespaceManager(blockEncodingSerde, functionsConfig, types, this); this.functionNamespaceManagers.put(JAVA_BUILTIN_NAMESPACE.getCatalogName(), builtInTypeAndFunctionNamespaceManager); this.functionInvokerProvider = new FunctionInvokerProvider(this); @@ -449,6 +453,16 @@ public void registerPluginFunctions(List functions) builtInPluginFunctionNamespaceManager.registerPluginFunctions(functions); } + public void registerConnectorFunctions(String catalogName, List functions) + { + FunctionNamespaceManager builtInPluginFunctionNamespaceManager = functionNamespaceManagers.get(catalogName); + if (builtInPluginFunctionNamespaceManager == null) { + builtInPluginFunctionNamespaceManager = new BuiltInTypeAndFunctionNamespaceManager(blockEncodingSerde, functionsConfig, types, this, false); + addFunctionNamespace(catalogName, builtInPluginFunctionNamespaceManager); + } + ((BuiltInTypeAndFunctionNamespaceManager) builtInPluginFunctionNamespaceManager).registerBuiltInFunctions(functions); + } + /** * likePattern / escape is an opportunistic optimization push down to function namespace managers. * Not all function namespace managers can handle it, thus the returned function list could diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionExtractor.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionExtractor.java index 7dc7ee5194249..7cdbd512a7ce3 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionExtractor.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/FunctionExtractor.java @@ -32,15 +32,21 @@ import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.lang.String.format; public final class FunctionExtractor { private FunctionExtractor() {} public static List extractFunctions(Collection> classes) + { + return extractFunctions(classes, JAVA_BUILTIN_NAMESPACE); + } + + public static List extractFunctions(Collection> classes, CatalogSchemaName functionNamespace) { return classes.stream() - .map(FunctionExtractor::extractFunctions) + .map(c -> extractFunctions(c, functionNamespace)) .flatMap(Collection::stream) .collect(toImmutableList()); } @@ -53,17 +59,21 @@ public static List extractFunctions(Class clazz) public static List extractFunctions(Class clazz, CatalogSchemaName defaultNamespace) { if (WindowFunction.class.isAssignableFrom(clazz)) { + checkArgument(defaultNamespace.equals(JAVA_BUILTIN_NAMESPACE), format("Connector specific Window functions are not supported: Class [%s], Namespace [%s]", clazz.getName(), defaultNamespace)); @SuppressWarnings("unchecked") Class windowClazz = (Class) clazz; return WindowAnnotationsParser.parseFunctionDefinition(windowClazz); } if (clazz.isAnnotationPresent(AggregationFunction.class)) { - return SqlAggregationFunction.createFunctionsByAnnotations(clazz); + return SqlAggregationFunction.createFunctionsByAnnotations(clazz, defaultNamespace); } - if (clazz.isAnnotationPresent(ScalarFunction.class) || - clazz.isAnnotationPresent(ScalarOperator.class)) { + if (clazz.isAnnotationPresent(ScalarFunction.class)) { + return ScalarFromAnnotationsParser.parseFunctionDefinition(clazz, defaultNamespace); + } + if (clazz.isAnnotationPresent(ScalarOperator.class)) { + checkArgument(defaultNamespace.equals(JAVA_BUILTIN_NAMESPACE), format("Connector specific Scalar Operator functions are not supported: Class [%s], Namespace [%s]", clazz.getName(), defaultNamespace)); return ScalarFromAnnotationsParser.parseFunctionDefinition(clazz); } @@ -72,9 +82,9 @@ public static List extractFunctions(Class clazz, Catal } List scalarFunctions = ImmutableList.builder() - .addAll(ScalarFromAnnotationsParser.parseFunctionDefinitions(clazz)) + .addAll(ScalarFromAnnotationsParser.parseFunctionDefinitions(clazz, defaultNamespace)) .addAll(SqlInvokedScalarFromAnnotationsParser.parseFunctionDefinitions(clazz, defaultNamespace)) - .addAll(CodegenScalarFromAnnotationsParser.parseFunctionDefinitions(clazz)) + .addAll(CodegenScalarFromAnnotationsParser.parseFunctionDefinitions(clazz, defaultNamespace)) .build(); checkArgument(!scalarFunctions.isEmpty(), "Class [%s] does not define any scalar functions", clazz.getName()); return scalarFunctions; diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/Metadata.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/Metadata.java index 3040846902745..f7283b436c65f 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/Metadata.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/Metadata.java @@ -72,6 +72,8 @@ public interface Metadata void registerBuiltInFunctions(List functions); + void registerConnectorFunctions(String catalogName, List functionInfos); + List listSchemaNames(Session session, String catalogName); /** diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/MetadataManager.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/MetadataManager.java index b6a3a644a7f50..6da7be2151bf5 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/MetadataManager.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/MetadataManager.java @@ -302,6 +302,12 @@ public void registerBuiltInFunctions(List functionInfos) functionAndTypeManager.registerBuiltInFunctions(functionInfos); } + @Override + public void registerConnectorFunctions(String catalogName, List functionInfos) + { + functionAndTypeManager.registerConnectorFunctions(catalogName, functionInfos); + } + @Override public List listSchemaNames(Session session, String catalogName) { diff --git a/presto-main-base/src/main/java/com/facebook/presto/metadata/SqlAggregationFunction.java b/presto-main-base/src/main/java/com/facebook/presto/metadata/SqlAggregationFunction.java index 9d2b6c0273632..38f8f0e9a8ef8 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/metadata/SqlAggregationFunction.java +++ b/presto-main-base/src/main/java/com/facebook/presto/metadata/SqlAggregationFunction.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.metadata; +import com.facebook.presto.common.CatalogSchemaName; import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.operator.aggregation.AggregationFromAnnotationsParser; @@ -46,7 +47,12 @@ public static List createFunctionByAnnotations(Class public static List createFunctionsByAnnotations(Class aggregationDefinition) { - return AggregationFromAnnotationsParser.parseFunctionDefinitions(aggregationDefinition) + return createFunctionsByAnnotations(aggregationDefinition, JAVA_BUILTIN_NAMESPACE); + } + + public static List createFunctionsByAnnotations(Class aggregationDefinition, CatalogSchemaName functionNamespace) + { + return AggregationFromAnnotationsParser.parseFunctionDefinitions(aggregationDefinition, functionNamespace) .stream() .map(x -> (SqlAggregationFunction) x) .collect(toImmutableList()); diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AggregationFromAnnotationsParser.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AggregationFromAnnotationsParser.java index 0595a555c4efc..696303dcb014c 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AggregationFromAnnotationsParser.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AggregationFromAnnotationsParser.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.aggregation; +import com.facebook.presto.common.CatalogSchemaName; import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.operator.ParametricImplementationsGroup; import com.facebook.presto.operator.annotations.FunctionsParserHelper; @@ -34,6 +35,7 @@ import java.util.Optional; import java.util.Set; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.aggregation.AggregationImplementation.Parser.parseImplementation; import static com.facebook.presto.operator.annotations.FunctionsParserHelper.parseDescription; import static com.google.common.base.Preconditions.checkArgument; @@ -54,7 +56,7 @@ public static ParametricAggregation parseFunctionDefinitionWithTypesConstraint(C { requireNonNull(returnType, "returnType is null"); requireNonNull(argumentTypes, "argumentTypes is null"); - for (ParametricAggregation aggregation : parseFunctionDefinitions(clazz)) { + for (ParametricAggregation aggregation : parseFunctionDefinitions(clazz, JAVA_BUILTIN_NAMESPACE)) { if (aggregation.getSignature().getReturnType().equals(returnType) && aggregation.getSignature().getArgumentTypes().equals(argumentTypes)) { return aggregation; @@ -64,6 +66,11 @@ public static ParametricAggregation parseFunctionDefinitionWithTypesConstraint(C } public static List parseFunctionDefinitions(Class aggregationDefinition) + { + return parseFunctionDefinitions(aggregationDefinition, JAVA_BUILTIN_NAMESPACE); + } + + public static List parseFunctionDefinitions(Class aggregationDefinition, CatalogSchemaName functionNamespace) { AggregationFunction aggregationAnnotation = aggregationDefinition.getAnnotation(AggregationFunction.class); requireNonNull(aggregationAnnotation, "aggregationAnnotation is null"); @@ -76,7 +83,7 @@ public static List parseFunctionDefinitions(Class aggr for (Method outputFunction : getOutputFunctions(aggregationDefinition, stateClass)) { for (Method inputFunction : getInputFunctions(aggregationDefinition, stateClass)) { for (AggregationHeader header : parseHeaders(aggregationDefinition, outputFunction)) { - AggregationImplementation onlyImplementation = parseImplementation(aggregationDefinition, header, stateClass, inputFunction, outputFunction, combineFunction, aggregationStateSerializerFactory); + AggregationImplementation onlyImplementation = parseImplementation(aggregationDefinition, header, stateClass, inputFunction, outputFunction, combineFunction, aggregationStateSerializerFactory, functionNamespace); ParametricImplementationsGroup implementations = ParametricImplementationsGroup.of(onlyImplementation); builder.add(new ParametricAggregation(implementations.getSignature(), header, implementations)); } @@ -97,7 +104,7 @@ public static ParametricAggregation parseFunctionDefinition(Class aggregation Optional aggregationStateSerializerFactory = getAggregationStateSerializerFactory(aggregationDefinition, stateClass); Method outputFunction = getOnlyElement(getOutputFunctions(aggregationDefinition, stateClass)); for (Method inputFunction : getInputFunctions(aggregationDefinition, stateClass)) { - AggregationImplementation implementation = parseImplementation(aggregationDefinition, header, stateClass, inputFunction, outputFunction, combineFunction, aggregationStateSerializerFactory); + AggregationImplementation implementation = parseImplementation(aggregationDefinition, header, stateClass, inputFunction, outputFunction, combineFunction, aggregationStateSerializerFactory, JAVA_BUILTIN_NAMESPACE); implementationsBuilder.addImplementation(implementation); } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AggregationImplementation.java b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AggregationImplementation.java index 8cb42304cd67f..5ffbd15f4d3de 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AggregationImplementation.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/aggregation/AggregationImplementation.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.aggregation; +import com.facebook.presto.common.CatalogSchemaName; import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.block.Block; import com.facebook.presto.common.function.SqlFunctionProperties; @@ -46,7 +47,6 @@ import java.util.stream.Stream; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.annotations.FunctionsParserHelper.containsAnnotation; import static com.facebook.presto.operator.annotations.FunctionsParserHelper.createTypeVariableConstraints; import static com.facebook.presto.operator.annotations.FunctionsParserHelper.parseLiteralParameters; @@ -246,6 +246,7 @@ public static final class Parser private final AggregationHeader header; private final Set literalParameters; private final List typeParameters; + private final CatalogSchemaName functionNamespace; private Parser( Class aggregationDefinition, @@ -254,7 +255,8 @@ private Parser( Method inputFunction, Method outputFunction, Method combineFunction, - Optional stateSerializerFactoryFunction) + Optional stateSerializerFactoryFunction, + CatalogSchemaName functionNamespace) { // rewrite data passed directly this.aggregationDefinition = aggregationDefinition; @@ -301,12 +303,13 @@ private Parser( inputHandle = methodHandle(inputFunction); combineHandle = methodHandle(combineFunction); outputHandle = methodHandle(outputFunction); + this.functionNamespace = requireNonNull(functionNamespace, "functionNamespace is null"); } private AggregationImplementation get() { Signature signature = new Signature( - QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, header.getName()), + QualifiedObjectName.valueOf(functionNamespace, header.getName()), FunctionKind.AGGREGATE, typeVariableConstraints, longVariableConstraints, @@ -336,9 +339,10 @@ public static AggregationImplementation parseImplementation( Method inputFunction, Method outputFunction, Method combineFunction, - Optional stateSerializerFactoryFunction) + Optional stateSerializerFactoryFunction, + CatalogSchemaName functionNamespace) { - return new Parser(aggregationDefinition, header, stateClass, inputFunction, outputFunction, combineFunction, stateSerializerFactoryFunction).get(); + return new Parser(aggregationDefinition, header, stateClass, inputFunction, outputFunction, combineFunction, stateSerializerFactoryFunction, functionNamespace).get(); } private static List parseParameterMetadataTypes(Method method) diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/CodegenScalarFromAnnotationsParser.java b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/CodegenScalarFromAnnotationsParser.java index 8ce2bab423946..ae9464001d024 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/CodegenScalarFromAnnotationsParser.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/CodegenScalarFromAnnotationsParser.java @@ -14,6 +14,7 @@ package com.facebook.presto.operator.scalar.annotations; +import com.facebook.presto.common.CatalogSchemaName; import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.type.Type; import com.facebook.presto.common.type.TypeSignature; @@ -70,9 +71,14 @@ public class CodegenScalarFromAnnotationsParser private CodegenScalarFromAnnotationsParser() {} public static List parseFunctionDefinitions(Class clazz) + { + return parseFunctionDefinitions(clazz, JAVA_BUILTIN_NAMESPACE); + } + + public static List parseFunctionDefinitions(Class clazz, CatalogSchemaName functionNamespace) { return findScalarsInFunctionDefinitionClass(clazz).stream() - .map(method -> createSqlScalarFunction(method)) + .map(method -> createSqlScalarFunction(method, functionNamespace)) .collect(toImmutableList()); } @@ -112,12 +118,12 @@ private static List getArgumentProperties(Method method) .collect(toImmutableList()); } - private static SqlScalarFunction createSqlScalarFunction(Method method) + private static SqlScalarFunction createSqlScalarFunction(Method method, CatalogSchemaName functionNamespace) { CodegenScalarFunction codegenScalarFunction = method.getAnnotation(CodegenScalarFunction.class); Signature signature = new Signature( - QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, codegenScalarFunction.value()), + QualifiedObjectName.valueOf(functionNamespace, codegenScalarFunction.value()), FunctionKind.SCALAR, Arrays.stream(method.getAnnotationsByType(TypeParameter.class)).map(t -> withVariadicBound(t.value(), t.boundedBy().isEmpty() ? null : t.boundedBy())).collect(toImmutableList()), ImmutableList.of(), diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarFromAnnotationsParser.java b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarFromAnnotationsParser.java index bc5970d90e9b9..a0b04fd003cd5 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarFromAnnotationsParser.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarFromAnnotationsParser.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.scalar.annotations; +import com.facebook.presto.common.CatalogSchemaName; import com.facebook.presto.metadata.SqlScalarFunction; import com.facebook.presto.operator.ParametricImplementationsGroup; import com.facebook.presto.operator.annotations.FunctionsParserHelper; @@ -36,11 +37,13 @@ import java.util.Optional; import java.util.Set; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.annotations.FunctionsParserHelper.checkPushdownSubfieldArgIndex; import static com.facebook.presto.operator.scalar.annotations.OperatorValidator.validateOperator; import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_ERROR; import static com.facebook.presto.util.Failures.checkCondition; import static com.google.common.base.Preconditions.checkArgument; +import static java.lang.String.format; import static java.util.Objects.requireNonNull; public final class ScalarFromAnnotationsParser @@ -48,28 +51,38 @@ public final class ScalarFromAnnotationsParser private ScalarFromAnnotationsParser() {} public static List parseFunctionDefinition(Class clazz) + { + return parseFunctionDefinition(clazz, JAVA_BUILTIN_NAMESPACE); + } + + public static List parseFunctionDefinition(Class clazz, CatalogSchemaName functionNamespace) { ImmutableList.Builder builder = ImmutableList.builder(); - for (ScalarHeaderAndMethods scalar : findScalarsInFunctionDefinitionClass(clazz)) { + for (ScalarHeaderAndMethods scalar : findScalarsInFunctionDefinitionClass(clazz, functionNamespace)) { builder.add(parseParametricScalar(scalar, FunctionsParserHelper.findConstructor(clazz))); } return builder.build(); } public static List parseFunctionDefinitions(Class clazz) + { + return parseFunctionDefinitions(clazz, JAVA_BUILTIN_NAMESPACE); + } + + public static List parseFunctionDefinitions(Class clazz, CatalogSchemaName functionNamespace) { ImmutableList.Builder builder = ImmutableList.builder(); - for (ScalarHeaderAndMethods methods : findScalarsInFunctionSetClass(clazz)) { + for (ScalarHeaderAndMethods methods : findScalarsInFunctionSetClass(clazz, functionNamespace)) { // Non-static function only makes sense in classes annotated @ScalarFunction. builder.add(parseParametricScalar(methods, Optional.empty())); } return builder.build(); } - private static List findScalarsInFunctionDefinitionClass(Class annotated) + private static List findScalarsInFunctionDefinitionClass(Class annotated, CatalogSchemaName functionNamespace) { ImmutableList.Builder builder = ImmutableList.builder(); - List classHeaders = ScalarImplementationHeader.fromAnnotatedElement(annotated); + List classHeaders = ScalarImplementationHeader.fromAnnotatedElement(annotated, functionNamespace); checkArgument(!classHeaders.isEmpty(), "Class [%s] that defines function must be annotated with @ScalarFunction or @ScalarOperator", annotated.getName()); for (ScalarImplementationHeader header : classHeaders) { @@ -85,7 +98,7 @@ private static List findScalarsInFunctionDefinitionClass return builder.build(); } - private static List findScalarsInFunctionSetClass(Class annotated) + private static List findScalarsInFunctionSetClass(Class annotated, CatalogSchemaName functionNamespace) { ImmutableList.Builder builder = ImmutableList.builder(); for (Method method : FunctionsParserHelper.findPublicMethods( @@ -94,7 +107,10 @@ private static List findScalarsInFunctionSetClass(Class< ImmutableSet.of(SqlInvokedScalarFunction.class, CodegenScalarFunction.class))) { checkCondition((method.getAnnotation(ScalarFunction.class) != null) || (method.getAnnotation(ScalarOperator.class) != null), FUNCTION_IMPLEMENTATION_ERROR, "Method [%s] annotated with @SqlType is missing @ScalarFunction or @ScalarOperator", method); - for (ScalarImplementationHeader header : ScalarImplementationHeader.fromAnnotatedElement(method)) { + if (method.getAnnotation(ScalarOperator.class) != null) { + checkArgument(functionNamespace.equals(JAVA_BUILTIN_NAMESPACE), format("Connector specific Scalar operator functions are not supported: Class [%s], Namespace [%s]", annotated.getName(), functionNamespace)); + } + for (ScalarImplementationHeader header : ScalarImplementationHeader.fromAnnotatedElement(method, functionNamespace)) { builder.add(new ScalarHeaderAndMethods(header, ImmutableSet.of(method))); } } diff --git a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarImplementationHeader.java b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarImplementationHeader.java index 2a3045bed71b1..101225c5dc1af 100644 --- a/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarImplementationHeader.java +++ b/presto-main-base/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarImplementationHeader.java @@ -13,6 +13,7 @@ */ package com.facebook.presto.operator.scalar.annotations; +import com.facebook.presto.common.CatalogSchemaName; import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.function.OperatorType; import com.facebook.presto.operator.scalar.ScalarHeader; @@ -44,7 +45,12 @@ public class ScalarImplementationHeader private ScalarImplementationHeader(String name, ScalarHeader header) { - this.name = QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, requireNonNull(name)); + this(name, header, JAVA_BUILTIN_NAMESPACE); + } + + private ScalarImplementationHeader(String name, ScalarHeader header, CatalogSchemaName functionNamespace) + { + this.name = QualifiedObjectName.valueOf(requireNonNull(functionNamespace), requireNonNull(name)); this.operatorType = Optional.empty(); this.header = requireNonNull(header); } @@ -73,7 +79,7 @@ private static String camelToSnake(String name) return LOWER_CAMEL.to(LOWER_UNDERSCORE, name); } - public static List fromAnnotatedElement(AnnotatedElement annotated) + public static List fromAnnotatedElement(AnnotatedElement annotated, CatalogSchemaName functionNamespace) { ScalarFunction scalarFunction = annotated.getAnnotation(ScalarFunction.class); ScalarOperator scalarOperator = annotated.getAnnotation(ScalarOperator.class); @@ -84,10 +90,10 @@ public static List fromAnnotatedElement(AnnotatedEle if (scalarFunction != null) { String baseName = scalarFunction.value().isEmpty() ? camelToSnake(annotatedName(annotated)) : scalarFunction.value(); - builder.add(new ScalarImplementationHeader(baseName, new ScalarHeader(description, scalarFunction.visibility(), scalarFunction.deterministic(), scalarFunction.calledOnNullInput(), descriptor))); + builder.add(new ScalarImplementationHeader(baseName, new ScalarHeader(description, scalarFunction.visibility(), scalarFunction.deterministic(), scalarFunction.calledOnNullInput(), descriptor), functionNamespace)); for (String alias : scalarFunction.alias()) { - builder.add(new ScalarImplementationHeader(alias, new ScalarHeader(description, scalarFunction.visibility(), scalarFunction.deterministic(), scalarFunction.calledOnNullInput(), descriptor))); + builder.add(new ScalarImplementationHeader(alias, new ScalarHeader(description, scalarFunction.visibility(), scalarFunction.deterministic(), scalarFunction.calledOnNullInput(), descriptor), functionNamespace)); } } diff --git a/presto-main-base/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java b/presto-main-base/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java index 79b8cd8300d58..e13cbb5a80c0c 100644 --- a/presto-main-base/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java +++ b/presto-main-base/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java @@ -85,6 +85,12 @@ public void registerBuiltInFunctions(List functions) throw new UnsupportedOperationException(); } + @Override + public void registerConnectorFunctions(String catalogName, List functionInfos) + { + throw new UnsupportedOperationException(); + } + @Override public MetadataResolver getMetadataResolver(Session session) { diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java b/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java index 0164dd10c3af1..b32417367c437 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/Plugin.java @@ -61,6 +61,7 @@ default Iterable getParametricTypes() return emptyList(); } + // getFunctions will be deprecated soon. Use Connector->getSystemFunctions() to implement connector level functions default Set> getFunctions() { return emptySet(); diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/connector/Connector.java b/presto-spi/src/main/java/com/facebook/presto/spi/connector/Connector.java index 6fd46075ff1c8..3d03bc7a324c4 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/connector/Connector.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/connector/Connector.java @@ -109,6 +109,14 @@ default Set getProcedures() return emptySet(); } + /** + * @return the set of functions provided by this connector + */ + default Set> getSystemFunctions() + { + return emptySet(); + } + /** * @return the system properties for this connector */ diff --git a/presto-tests/src/test/java/com/facebook/presto/functions/TestBuiltInConnectorFunctions.java b/presto-tests/src/test/java/com/facebook/presto/functions/TestBuiltInConnectorFunctions.java new file mode 100644 index 0000000000000..3cf1c2199b9c3 --- /dev/null +++ b/presto-tests/src/test/java/com/facebook/presto/functions/TestBuiltInConnectorFunctions.java @@ -0,0 +1,231 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.functions; + +import com.facebook.presto.common.RuntimeStats; +import com.facebook.presto.common.type.TimeZoneKey; +import com.facebook.presto.common.type.Type; +import com.facebook.presto.common.type.TypeManager; +import com.facebook.presto.operator.aggregation.CountAggregation; +import com.facebook.presto.operator.scalar.MapFilterFunction; +import com.facebook.presto.operator.scalar.sql.MapNormalizeFunction; +import com.facebook.presto.operator.scalar.sql.StringSqlFunctions; +import com.facebook.presto.server.testing.TestingPrestoServer; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorHandleResolver; +import com.facebook.presto.spi.ConnectorPageSource; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.ConnectorSplit; +import com.facebook.presto.spi.ConnectorTableLayoutHandle; +import com.facebook.presto.spi.FixedPageSource; +import com.facebook.presto.spi.Plugin; +import com.facebook.presto.spi.SplitContext; +import com.facebook.presto.spi.connector.Connector; +import com.facebook.presto.spi.connector.ConnectorContext; +import com.facebook.presto.spi.connector.ConnectorFactory; +import com.facebook.presto.spi.connector.ConnectorMetadata; +import com.facebook.presto.spi.connector.ConnectorPageSinkProvider; +import com.facebook.presto.spi.connector.ConnectorPageSourceProvider; +import com.facebook.presto.spi.connector.ConnectorSplitManager; +import com.facebook.presto.spi.connector.ConnectorTransactionHandle; +import com.facebook.presto.spi.transaction.IsolationLevel; +import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.testing.TestingHandleResolver; +import com.facebook.presto.testing.TestingMetadata; +import com.facebook.presto.testing.TestingPageSinkProvider; +import com.facebook.presto.tests.TestingPrestoClient; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.inject.Key; +import org.intellij.lang.annotations.Language; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.common.type.DoubleType.DOUBLE; +import static com.facebook.presto.common.type.IntegerType.INTEGER; +import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static com.facebook.presto.util.StructuralTestUtil.mapType; +import static java.lang.String.format; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.fail; + +public class TestBuiltInConnectorFunctions +{ + protected TestingPrestoServer server; + protected TestingPrestoClient client; + protected TypeManager typeManager; + + private static final String PLUGIN_NAME = "test-function"; + private static final String CATALOG_NAME = "test_function_catalog"; + private static final String SCHEMA_NAME = "system"; + private static final String FUNCTION_NAMESPACE = CATALOG_NAME + "." + SCHEMA_NAME; + + @BeforeClass + public void setup() + throws Exception + { + server = new TestingPrestoServer(); + server.installPlugin(new TestConnectorWithBuiltinFunctions()); + server.createCatalog(CATALOG_NAME, PLUGIN_NAME); + client = new TestingPrestoClient(server, testSessionBuilder() + .setTimeZoneKey(TimeZoneKey.getTimeZoneKey("America/Bahia_Banderas")) + .build()); + typeManager = server.getInstance(Key.get(TypeManager.class)); + } + + public void assertInvalidFunction(String expr, String exceptionPattern) + { + try { + client.execute("SELECT " + expr); + fail("Function expected to fail but not"); + } + catch (Exception e) { + if (!(e.getMessage().matches(exceptionPattern))) { + fail(format("Expected exception message '%s' to match '%s' but not", + e.getMessage(), exceptionPattern)); + } + } + } + + private class TestConnectorWithBuiltinFunctions + implements Plugin + { + @Override + public Iterable getConnectorFactories() + { + return ImmutableList.of(new ConnectorFactory() + { + @Override + public String getName() + { + return PLUGIN_NAME; + } + + @Override + public ConnectorHandleResolver getHandleResolver() + { + return new TestingHandleResolver(); + } + + @Override + public Connector create(String catalogName, Map config, ConnectorContext context) + { + return new Connector() + { + private final ConnectorMetadata metadata = new TestingMetadata(); + + @Override + public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly) + { + return new ConnectorTransactionHandle() {}; + } + + @Override + public ConnectorMetadata getMetadata(ConnectorTransactionHandle transaction) + { + return metadata; + } + + @Override + public Set> getSystemFunctions() + { + return ImmutableSet.of(TestConnectorSystemFunctions.class, + TestConnectorSystemFunctions.SumFunction.class, + CountAggregation.class, + StringSqlFunctions.class, + MapNormalizeFunction.class, + MapFilterFunction.class); + } + + @Override + public ConnectorSplitManager getSplitManager() + { + return (transactionHandle, session, layout, splitSchedulingContext) -> { + throw new UnsupportedOperationException(); + }; + } + + @Override + public ConnectorPageSourceProvider getPageSourceProvider() + { + return new ConnectorPageSourceProvider() { + @Override + public ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorSplit split, ConnectorTableLayoutHandle layout, List columns, SplitContext splitContext, RuntimeStats runtimeStats) + { + return new FixedPageSource(ImmutableList.of()); + } + }; + } + + @Override + public ConnectorPageSinkProvider getPageSinkProvider() + { + return new TestingPageSinkProvider(); + } + }; + } + }); + } + } + + public void check(@Language("SQL") String query, Type expectedType, Object expectedValue) + { + MaterializedResult result = client.execute(query).getResult(); + assertEquals(result.getRowCount(), 1); + assertEquals(result.getTypes().get(0), expectedType); + Object actual = result.getMaterializedRows().get(0).getField(0); + assertEquals(actual, expectedValue); + } + + @Test + public void testNewFunctionNamespaceFunctions() + { + // Scalar functions with methods annotated with @ScalarFunction + check("SELECT " + FUNCTION_NAMESPACE + ".modulo(10,3)", BIGINT, 1L); + check("SELECT " + FUNCTION_NAMESPACE + ".identity('test-functions')", VARCHAR, "test-functions"); + + // Scalar function with class annotated with @ScalarFunction + check("SELECT " + FUNCTION_NAMESPACE + ".sum(10,3)", BIGINT, 13L); + + // Aggregation function + check("SELECT " + FUNCTION_NAMESPACE + ".count(*) from (values (0), (1), (2)) T(i)", BIGINT, 3L); + + // Sql invoked scalar function with class annotated with @SqlInvokedScalarFunction + check("SELECT " + FUNCTION_NAMESPACE + ".map_normalize(map(array['w', 'x', 'y', 'z'], array[1, 1, 1, 1]))", + mapType(VARCHAR, DOUBLE), + ImmutableMap.of("w", 0.25, "x", 0.25, "y", 0.25, "z", 0.25)); + + // Sql invoked scalar function with method annotated with @SqlInvokedScalarFunction + check("SELECT " + FUNCTION_NAMESPACE + ".trail('random_string_test', 4)", VARCHAR, "test"); + + // Code gen scalar function + check("SELECT " + FUNCTION_NAMESPACE + ".map_filter(map(ARRAY [5, 6, 7, 8], ARRAY [5, 6, 6, 5]), (x, y) -> x <= 6 OR y = 5)", + mapType(INTEGER, INTEGER), + ImmutableMap.of(5, 5, 6, 6, 8, 5)); + } + + @Test + public void testInvalidFunctionAndNamespace() + { + assertInvalidFunction(CATALOG_NAME + ".namespace.modulo(10,3)", "line 1:8: Function test_function_catalog.namespace.modulo not registered"); + assertInvalidFunction(CATALOG_NAME + ".system.some_func(10)", "line 1:8: Function test_function_catalog.system.some_func not registered"); + } +} diff --git a/presto-tests/src/test/java/com/facebook/presto/functions/TestConnectorSystemFunctions.java b/presto-tests/src/test/java/com/facebook/presto/functions/TestConnectorSystemFunctions.java new file mode 100644 index 0000000000000..ded1635c297c3 --- /dev/null +++ b/presto-tests/src/test/java/com/facebook/presto/functions/TestConnectorSystemFunctions.java @@ -0,0 +1,59 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.functions; + +import com.facebook.presto.common.type.StandardTypes; +import com.facebook.presto.spi.function.Description; +import com.facebook.presto.spi.function.ScalarFunction; +import com.facebook.presto.spi.function.SqlType; +import io.airlift.slice.Slice; + +public final class TestConnectorSystemFunctions +{ + private TestConnectorSystemFunctions() + {} + + @Description("Returns modulo of value by numberOfBuckets") + @ScalarFunction + @SqlType(StandardTypes.BIGINT) + public static long modulo( + @SqlType(StandardTypes.BIGINT) long value, + @SqlType(StandardTypes.BIGINT) long numberOfBuckets) + { + return value % numberOfBuckets; + } + + @Description(("Return the input string")) + @ScalarFunction + @SqlType(StandardTypes.VARCHAR) + public static Slice identity(@SqlType(StandardTypes.VARCHAR) Slice slice) + { + return slice; + } + + @ScalarFunction("sum") + public static class SumFunction + { + private SumFunction() + {} + @Description("Returns sum of two integers") + @SqlType(StandardTypes.BIGINT) + public static long sum( + @SqlType(StandardTypes.INTEGER) long a, + @SqlType(StandardTypes.INTEGER) long b) + { + return a + b; + } + } +}