diff --git a/pom.xml b/pom.xml index 003577d7c58a5..039a7ee27f6b0 100644 --- a/pom.xml +++ b/pom.xml @@ -177,6 +177,7 @@ presto-kudu presto-elasticsearch presto-function-namespace-managers + presto-function-namespace-managers-common presto-expressions presto-benchmark-runner presto-spark-classloader-interface @@ -1017,6 +1018,12 @@ ${project.version} + + com.facebook.presto + presto-native-sidecar-plugin + ${project.version} + + com.facebook.hive hive-dwrf diff --git a/presto-client/src/main/java/com/facebook/presto/client/FixJsonDataUtils.java b/presto-client/src/main/java/com/facebook/presto/client/FixJsonDataUtils.java index 25e4742f696ac..8a853726fb390 100644 --- a/presto-client/src/main/java/com/facebook/presto/client/FixJsonDataUtils.java +++ b/presto-client/src/main/java/com/facebook/presto/client/FixJsonDataUtils.java @@ -109,7 +109,7 @@ private static Object fixValue(TypeSignature signature, Object value) if (List.class.isAssignableFrom(value.getClass())) { List fixedValue = new ArrayList<>(); for (Object object : List.class.cast(value)) { - fixedValue.add(fixValue(signature.getTypeParametersAsTypeSignatures().get(0), object)); + fixedValue.add(fixValue(signature.getTypeOrNamedTypeParametersAsTypeSignatures().get(0), object)); } return fixedValue; } @@ -122,8 +122,8 @@ else if (value.getClass() == String.class) { } if (signature.getBase().equals(MAP)) { if (Map.class.isAssignableFrom(value.getClass())) { - TypeSignature keySignature = signature.getTypeParametersAsTypeSignatures().get(0); - TypeSignature valueSignature = signature.getTypeParametersAsTypeSignatures().get(1); + TypeSignature keySignature = signature.getTypeOrNamedTypeParametersAsTypeSignatures().get(0); + TypeSignature valueSignature = signature.getTypeOrNamedTypeParametersAsTypeSignatures().get(1); Map fixedValue = new HashMap<>(); for (Map.Entry entry : (Set>) Map.class.cast(value).entrySet()) { fixedValue.put(fixValue(keySignature, entry.getKey()), fixValue(valueSignature, entry.getValue())); diff --git a/presto-common/src/main/java/com/facebook/presto/common/type/ParameterKind.java b/presto-common/src/main/java/com/facebook/presto/common/type/ParameterKind.java index f36db1fc29082..1fb7a2bf71371 100644 --- a/presto-common/src/main/java/com/facebook/presto/common/type/ParameterKind.java +++ b/presto-common/src/main/java/com/facebook/presto/common/type/ParameterKind.java @@ -20,6 +20,38 @@ import java.util.Optional; +/** + * The {@code ParameterKind} enum represents various kinds of parameters used in Presto's type system. + * The available parameter kinds are: + * + *
    + *
  • TYPE: + * Used when the parameter itself is of type {@code TYPE_SIGNATURE}, representing a type definition.
    • + * + *
  • NAMED_TYPE: + * Represents parameters that are explicitly named and can be referenced using their name. + * This is primarily used when the base type is a row type.
    • + * + *
  • LONG: + * Used for types that take a long literal as a parameter. Examples include + * types like {@code decimal} and {@code varchar}.
    • + * + *
  • VARIABLE: + * Used when variables are passed as parameters. This allows dynamic and flexible parameter handling.
    • + * + *
  • LONG_ENUM: + * Represents a mapping of string values to long values. It is efficient for cases where + * symbolic names correspond to numeric values.
    • + * + *
  • VARCHAR_ENUM: + * Represents a mapping of string values to string values. This is useful for symbolic names + * that do not require numeric representation.
    • + * + *
  • DISTINCT_TYPE: + * Represents distinct user-defined types, enabling the creation of custom types in Presto's type system.
    • + *
+ */ + @ThriftEnum public enum ParameterKind { diff --git a/presto-common/src/main/java/com/facebook/presto/common/type/TypeSignature.java b/presto-common/src/main/java/com/facebook/presto/common/type/TypeSignature.java index 13f720f3891f9..7cdf0b91aee2f 100644 --- a/presto-common/src/main/java/com/facebook/presto/common/type/TypeSignature.java +++ b/presto-common/src/main/java/com/facebook/presto/common/type/TypeSignature.java @@ -133,17 +133,24 @@ public List getParameters() return parameters; } - public List getTypeParametersAsTypeSignatures() + public List getTypeOrNamedTypeParametersAsTypeSignatures() { List result = new ArrayList<>(); for (TypeSignatureParameter parameter : parameters) { - if (parameter.getKind() != ParameterKind.TYPE) { - throw new IllegalStateException( - format("Expected all parameters to be TypeSignatures but [%s] was found", parameter.toString())); + switch (parameter.getKind()) { + case TYPE: + result.add(parameter.getTypeSignature()); + break; + case NAMED_TYPE: + result.add(parameter.getNamedTypeSignature().getTypeSignature()); + break; + default: + throw new IllegalStateException( + format("Expected all parameters to be of kind TYPE or NAMED_TYPE but [%s] kind was found for parameter: [%s]", + parameter.getKind(), parameter)); } - result.add(parameter.getTypeSignature()); } - return result; + return unmodifiableList(result); } public boolean isCalculated() diff --git a/presto-common/src/main/java/com/facebook/presto/common/type/TypeSignatureUtils.java b/presto-common/src/main/java/com/facebook/presto/common/type/TypeSignatureUtils.java new file mode 100644 index 0000000000000..e1e4d67618f4b --- /dev/null +++ b/presto-common/src/main/java/com/facebook/presto/common/type/TypeSignatureUtils.java @@ -0,0 +1,158 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.common.type; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import static java.util.Collections.unmodifiableList; + +public final class TypeSignatureUtils +{ + private TypeSignatureUtils() {} + + public static TypeSignature resolveIntermediateType(TypeSignature actualTypeSignature, List actualTypeSignatureParameters, List expectedTypeSignatureParameters) + { + Map typeSignatureMap = getTypeSignatureMap(actualTypeSignatureParameters, expectedTypeSignatureParameters); + return resolveTypeSignatures(actualTypeSignature, typeSignatureMap).getTypeSignature(); + } + + private static Map getTypeSignatureMap(List parameters, List argumentTypes) + { + Map typeSignatureMap = new HashMap<>(); + if (argumentTypes.size() != parameters.size()) { + throw new IllegalStateException( + "Parameters size: " + parameters.size() + " and argumentTypes size: " + argumentTypes.size() + " do not match !"); + } + + for (int i = 0; i < argumentTypes.size(); i++) { + TypeSignature parameter = parameters.get(i); + TypeSignature argumentType = argumentTypes.get(i); + + // Realistically, there are only two cases, + // 1. When parameter.getParameters() is empty : + // - Eg: parameter type = generic type(T) and argumentType = array(double), we can directly put + // map.put(parameter, argumentType) there is no need to loop over the argumentType. + // - Eg: param type = non-generic type(bigint) , in this case argumentType will also be of the + // same non-generic type hence map.put(parameter, argumentType) is valid here too. + // 2. When parameter.getParameters() is not empty: + // - Eg: parameter type = generic type (array(T)) and argumentType = array(double), we recursively run + // this function until we reach condition 1. + // Example calls for parameter type = generic type (array(T)) and argumentType = array(double) + // Iteration 1: + // Parameter: + // Type signature base = array + // parameters = T + // ArgumentType : + // Type signature base = array + // parameters = double + // Iteration 2: + // Parameter: + // Type signature base = T + // parameters = empty + // ArgumentType : + // Type signature base = double + // parameters = empty + // return typeSignatureMap = {"T": "double"} + + // If parameter params are of type long e.g decimal(15, 2) or + // of type varchar e.g decimal(i4, i5), we don't need to recursively call the function on its params + if (parameter.getParameters().isEmpty() || !(areParametersTypeSignatureOrNamedTypedSignature(parameter.getParameters()))) { + typeSignatureMap.put(parameter, argumentType); + } + else { + typeSignatureMap.putAll(getTypeSignatureMap( + parameter.getTypeOrNamedTypeParametersAsTypeSignatures(), + argumentType.getTypeOrNamedTypeParametersAsTypeSignatures())); + } + } + return typeSignatureMap; + } + + // A utility function to resolve intermediate type signatures. + // Realistically, these are the different cases that we can face: + // 1. If there are no params/argTypes in the first call itself, return the resolvedType from the map directly. + // 2. If params != empty, we loop over the params: + // - Check whether the param is present in the typeSignatureMap, if its present, + // add the resolvedTypeParameterSignature mapping as a param directly and continue to the next param. + // - The idea behind this logic is that if param type is present in the typeSignatureMap, it means that for + // that particular param, we could just resolve the type from the map and no need to recursively call as its completely resolved. + // - Eg: param : T , map : {"T":"array(double)"}. + // - If the mapping isn't present, we recursively call the resolveTypeSignatures() again. + // - Example calls for param: array(T) , map : {"T" : "array(double)"} + // resolvedIntermediateType = null + // Iteration 1: + // Parameter: + // key to lookup in map: array(T), key found : false + // Iteration 2: + // Parameter: + // key to lookup in map: T, key found : true + // resolvedIntermediateType = array(double) + // return resolvedIntermediateType = array(array(double)) + // + // + private static NamedTypeSignature resolveTypeSignatures(TypeSignature typeSignature, Map typeSignatureMap) + { + if (typeSignatureMap.containsKey(typeSignature)) { + TypeSignature resolvedTypeSignature = typeSignatureMap.get(typeSignature); + return new NamedTypeSignature(Optional.empty(), resolvedTypeSignature); + } + List namedTypeSignatures = new ArrayList<>(); + List typeSignatures = new ArrayList<>(); + for (TypeSignature typeParameterSignature : typeSignature.getTypeOrNamedTypeParametersAsTypeSignatures()) { + // if base is "row" typeSignature, all typeParameterSignatures need to be of type NamedTypeSignature. + boolean isRowTypeSignatureBase = typeSignature.getBase().equals("row"); + if (typeSignatureMap.containsKey(typeParameterSignature)) { + TypeSignature resolvedTypeParameterSignature = typeSignatureMap.get(typeParameterSignature); + if (isRowTypeSignatureBase) { + namedTypeSignatures.add(new NamedTypeSignature(Optional.empty(), resolvedTypeParameterSignature)); + } + else { + typeSignatures.add(resolvedTypeParameterSignature); + } + } + else { + NamedTypeSignature namedTypeSignature = resolveTypeSignatures(typeParameterSignature, typeSignatureMap); + if (isRowTypeSignatureBase) { + namedTypeSignatures.add(namedTypeSignature); + } + else { + typeSignatures.add(namedTypeSignature.getTypeSignature()); + } + } + } + + List parameters; + if (!typeSignatures.isEmpty()) { + parameters = typeSignatures.stream().map(TypeSignatureParameter::of).collect(Collectors.toList()); + } + else { + parameters = namedTypeSignatures.stream().map(TypeSignatureParameter::of).collect(Collectors.toList()); + } + + return new NamedTypeSignature(Optional.empty(), new TypeSignature(typeSignature.getBase(), unmodifiableList(parameters))); + } + + private static boolean areParametersTypeSignatureOrNamedTypedSignature(List parameters) + { + return !parameters.isEmpty() && parameters.stream() + .map(TypeSignatureParameter::getKind) + .allMatch(parameterKind -> + parameterKind.equals(ParameterKind.NAMED_TYPE) || parameterKind.equals(ParameterKind.TYPE)); + } +} diff --git a/presto-common/src/test/java/com/facebook/presto/common/type/TestTypeSignatureUtils.java b/presto-common/src/test/java/com/facebook/presto/common/type/TestTypeSignatureUtils.java new file mode 100644 index 0000000000000..76befed7efc5a --- /dev/null +++ b/presto-common/src/test/java/com/facebook/presto/common/type/TestTypeSignatureUtils.java @@ -0,0 +1,473 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.common.type; + +import com.facebook.presto.common.QualifiedObjectName; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; +import static com.facebook.presto.common.type.TypeSignatureUtils.resolveIntermediateType; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; + +public class TestTypeSignatureUtils +{ + @Test + public void testResolveIntermediateTypeNonGenericBigintType() + { + TypeSignature actualIntermediateType = parseTypeSignature("bigint"); + TypeSignature expectedIntermediateType = parseTypeSignature("bigint"); + TypeSignature resolvedIntermediateType = resolveIntermediateType( + actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeNonGenericDifferentArgTypes() + { + TypeSignature actualIntermediateType = parseTypeSignature("bigint"); + TypeSignature expectedIntermediateType = parseTypeSignature("bigint"); + TypeSignature resolvedIntermediateType = resolveIntermediateType( + actualIntermediateType, + ImmutableList.of(parseTypeSignature("double")), + ImmutableList.of(parseTypeSignature("double"))); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeGenericBigintType() + { + TypeSignature actualIntermediateType = parseTypeSignature("T"); + TypeSignature expectedIntermediateType = parseTypeSignature("bigint"); + TypeSignature resolvedIntermediateType = resolveIntermediateType( + actualIntermediateType, + ImmutableList.of(actualIntermediateType), + ImmutableList.of(expectedIntermediateType)); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeGenericBigintArgType() + { + TypeSignature actualIntermediateType = parseTypeSignature("test(T)"); + TypeSignature expectedIntermediateType = parseTypeSignature("test(bigint)"); + TypeSignature resolvedIntermediateType = resolveIntermediateType( + actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeGenericMultipleArgs() + { + TypeSignature actualIntermediateType = parseTypeSignature("test(array(T),boolean,double,integer,E,array(T))"); + TypeSignature expectedIntermediateType = parseTypeSignature("test(array(bigint),boolean,double,integer,varchar(100), array(bigint))"); + TypeSignature resolvedIntermediateType = resolveIntermediateType( + actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeNonGenericRowType() + { + TypeSignature actualIntermediateType = parseTypeSignature("row(bigint)"); + TypeSignature expectedIntermediateType = parseTypeSignature("row(bigint)"); + TypeSignature resolvedIntermediateType = resolveIntermediateType( + actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeGenericRowType() + { + TypeSignature actualIntermediateType = parseTypeSignature("row(T)"); + TypeSignature expectedIntermediateType = parseTypeSignature("row(bigint)"); + TypeSignature resolvedIntermediateType = resolveIntermediateType( + actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeGenericRowArgTypeSingleParam() + { + TypeSignature actualIntermediateType = parseTypeSignature("test(row(T))"); + TypeSignature expectedIntermediateType = parseTypeSignature("test(row(bigint))"); + TypeSignature resolvedIntermediateType = resolveIntermediateType( + actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeGenericRowArgTypeMultipleParams() + { + TypeSignature actualIntermediateType = parseTypeSignature("test(row(array(T),boolean,double,integer,E,array(T)))"); + TypeSignature expectedIntermediateType = parseTypeSignature("test(row(array(bigint),boolean,double,integer,varchar(100), array(bigint)))"); + TypeSignature resolvedIntermediateType = resolveIntermediateType( + actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeGenericRowTypeMultipleParams() + { + TypeSignature actualIntermediateType = parseTypeSignature("row(array(T),boolean,double,integer,E,array(T),array(integer))"); + TypeSignature expectedIntermediateType = parseTypeSignature("row(array(bigint),boolean,double,integer,varchar(100),array(bigint),array(integer))"); + TypeSignature resolvedIntermediateType = resolveIntermediateType( + actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeNonGenericDecimalType() + { + TypeSignature actualIntermediateType = parseTypeSignature("decimal(15, 2)"); + TypeSignature expectedIntermediateType = parseTypeSignature("test(decimal(15, 2))"); + List expectedIntermediateTypeParameters = expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(); + TypeSignature resolvedIntermediateType = resolveIntermediateType( + actualIntermediateType, + ImmutableList.of(actualIntermediateType), + expectedIntermediateTypeParameters); + assertFalse(expectedIntermediateTypeParameters.isEmpty()); + assertEquals(resolvedIntermediateType, expectedIntermediateTypeParameters.get(0)); + } + + @Test + public void testResolveIntermediateTypeNonGenericDecimalArgType() + { + TypeSignature actualIntermediateType = parseTypeSignature("test(decimal(15, 2))"); + TypeSignature expectedIntermediateType = parseTypeSignature("test(decimal(15, 2))"); + TypeSignature resolvedIntermediateType = resolveIntermediateType( + actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeGenericMapArgType() + { + TypeSignature actualIntermediateType = parseTypeSignature("map(T, decimal(15, 2))"); + TypeSignature expectedIntermediateType = parseTypeSignature("map(integer, decimal(15, 2))"); + TypeSignature resolvedIntermediateType = resolveIntermediateType( + actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeGenericMapArgTypeMapParam() + { + TypeSignature actualIntermediateType = parseTypeSignature("map(map(array(K), V), decimal(15, 2))"); + TypeSignature expectedIntermediateType = parseTypeSignature("map(map(array(double), decimal(15, 2)), decimal(15, 2))"); + TypeSignature resolvedIntermediateType = resolveIntermediateType( + actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeGenericRowTypeMapParam() + { + TypeSignature actualIntermediateType = parseTypeSignature("row(map(map(array(K), V), decimal(15, 2)))"); + TypeSignature expectedIntermediateType = parseTypeSignature("row(map(map(array(double), varchar(50)), decimal(15, 2)))"); + TypeSignature resolvedIntermediateType = resolveIntermediateType( + actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeNonGenericRowTypeMapParam() + { + TypeSignature actualIntermediateType = parseTypeSignature("row(map(map(array(double), varchar(50)), decimal(15, 2)))"); + TypeSignature expectedIntermediateType = parseTypeSignature("row(map(map(array(double), varchar(50)), decimal(15, 2)))"); + TypeSignature resolvedIntermediateType = resolveIntermediateType( + actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeGenericFunctionType() + { + TypeSignature actualIntermediateType = parseTypeSignature("S"); + TypeSignature expectedIntermediateType = parseTypeSignature("array(bigint)"); + List actualIntermediateTypeParams = + parseTypeSignature("test(T, S, function(S, T, S), function(S, S, S))").getTypeOrNamedTypeParametersAsTypeSignatures(); + List expectedIntermediateTypeParams = + parseTypeSignature( + "test(integer, array(bigint), function(array(bigint), integer, array(bigint)), " + + "function(array(bigint), array(bigint), array(bigint)))").getTypeOrNamedTypeParametersAsTypeSignatures(); + TypeSignature resolvedIntermediateType = resolveIntermediateType( + actualIntermediateType, + actualIntermediateTypeParams, + expectedIntermediateTypeParams); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test(expectedExceptions = IllegalStateException.class, expectedExceptionsMessageRegExp = + "Parameters size: 7 and argumentTypes size: 6 do not match !") + public void testFailingResolveIntermediateTypeExtraParams() + { + TypeSignature actualIntermediateType = + parseTypeSignature("row(array(T),boolean,double,integer,E,array(T),array(integer))"); + TypeSignature expectedIntermediateType = + parseTypeSignature("row(array(bigint),boolean,double,integer,varchar(100),array(bigint))"); + TypeSignature resolvedIntermediateType = + resolveIntermediateType(actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test(expectedExceptions = IllegalStateException.class, expectedExceptionsMessageRegExp = + "Parameters size: 6 and argumentTypes size: 7 do not match !") + public void testFailingResolveIntermediateTypeExtraArgs() + { + TypeSignature actualIntermediateType = + parseTypeSignature("row(array(T),boolean,double,integer,E,array(T))"); + TypeSignature expectedIntermediateType = + parseTypeSignature("row(array(bigint),boolean,double,integer,varchar(100),array(bigint), array(integer))"); + TypeSignature resolvedIntermediateType = + resolveIntermediateType(actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeGenericEnumParametricType() + { + TypeSignature actualIntermediateType = + parseTypeSignature("test(T)"); + TypeSignature test = new TypeSignature( + StandardTypes.MAP, + TypeSignatureParameter.of((new VarcharEnumType(new VarcharEnumType.VarcharEnumMap("test.enum.my_enum", ImmutableMap.of("k", "v)))"))).getTypeSignature())), + TypeSignatureParameter.of(new BigintEnumType(new BigintEnumType.LongEnumMap("test.enum.my_enum_2", ImmutableMap.of("k", 1L))).getTypeSignature())); + TypeSignature expectedIntermediateType = new TypeSignature( + "test", + TypeSignatureParameter.of(test)); + TypeSignature resolvedIntermediateType = + resolveIntermediateType(actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeNonGenericEnumParametricType() + { + TypeSignature actualIntermediateType = new + TypeSignature("test", + TypeSignatureParameter.of( + new TypeSignature( + StandardTypes.MAP, + TypeSignatureParameter.of((new VarcharEnumType(new VarcharEnumType.VarcharEnumMap("test.enum.my_enum", ImmutableMap.of("k", "v)))"))).getTypeSignature())), + TypeSignatureParameter.of(new BigintEnumType(new BigintEnumType.LongEnumMap("test.enum.my_enum_2", ImmutableMap.of("k", 1L))).getTypeSignature())))); + + TypeSignature expectedIntermediateType = new + TypeSignature("test", + TypeSignatureParameter.of( + new TypeSignature( + StandardTypes.MAP, + TypeSignatureParameter.of((new VarcharEnumType(new VarcharEnumType.VarcharEnumMap("test.enum.my_enum", ImmutableMap.of("k", "v)))"))).getTypeSignature())), + TypeSignatureParameter.of(new BigintEnumType(new BigintEnumType.LongEnumMap("test.enum.my_enum_2", ImmutableMap.of("k", 1L))).getTypeSignature())))); + + TypeSignature resolvedIntermediateType = + resolveIntermediateType(actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeNonGenericRowEnumParametricType() + { + TypeSignature actualIntermediateType = new + TypeSignature("row", + TypeSignatureParameter.of(new NamedTypeSignature(Optional.empty(), parseTypeSignature("double"))), + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.empty(), + new TypeSignature( + StandardTypes.MAP, + TypeSignatureParameter.of((new VarcharEnumType(new VarcharEnumType.VarcharEnumMap("test.enum.my_enum", ImmutableMap.of("k", "v)))"))).getTypeSignature())), + TypeSignatureParameter.of(new BigintEnumType(new BigintEnumType.LongEnumMap("test.enum.my_enum_2", ImmutableMap.of("k", 1L))).getTypeSignature()))))); + + TypeSignature expectedIntermediateType = new + TypeSignature("row", + TypeSignatureParameter.of(new NamedTypeSignature(Optional.empty(), parseTypeSignature("double"))), + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.empty(), + new TypeSignature( + StandardTypes.MAP, + TypeSignatureParameter.of((new VarcharEnumType(new VarcharEnumType.VarcharEnumMap("test.enum.my_enum", ImmutableMap.of("k", "v)))"))).getTypeSignature())), + TypeSignatureParameter.of(new BigintEnumType(new BigintEnumType.LongEnumMap("test.enum.my_enum_2", ImmutableMap.of("k", 1L))).getTypeSignature()))))); + + TypeSignature resolvedIntermediateType = + resolveIntermediateType(actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeGenericRowEnumParametricType() + { + TypeSignature actualIntermediateType = new + TypeSignature("row", + TypeSignatureParameter.of(new NamedTypeSignature(Optional.empty(), parseTypeSignature("K"))), + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.empty(), + new TypeSignature( + StandardTypes.MAP, + TypeSignatureParameter.of((new VarcharEnumType(new VarcharEnumType.VarcharEnumMap("test.enum.my_enum", ImmutableMap.of("k", "v)))"))).getTypeSignature())), + TypeSignatureParameter.of(parseTypeSignature("T")))))); + + TypeSignature expectedIntermediateType = new + TypeSignature("row", + TypeSignatureParameter.of(new NamedTypeSignature(Optional.empty(), parseTypeSignature("array(double)"))), + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.empty(), + new TypeSignature( + StandardTypes.MAP, + TypeSignatureParameter.of((new VarcharEnumType(new VarcharEnumType.VarcharEnumMap("test.enum.my_enum", ImmutableMap.of("k", "v)))"))).getTypeSignature())), + TypeSignatureParameter.of(new BigintEnumType(new BigintEnumType.LongEnumMap("test.enum.my_enum_2", ImmutableMap.of("k", 1L))).getTypeSignature()))))); + + TypeSignature resolvedIntermediateType = + resolveIntermediateType(actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeNonGenericDistinctParametricType() + { + TypeSignature actualIntermediateType = new + TypeSignature("test", + TypeSignatureParameter.of(parseTypeSignature("bigint")), + TypeSignatureParameter.of( + createTypeSignatureWithDistinctParameterKind("test.dt.int00", Optional.empty(), "integer"))); + + TypeSignature expectedIntermediateType = new + TypeSignature("test", + TypeSignatureParameter.of(parseTypeSignature("bigint")), + TypeSignatureParameter.of( + createTypeSignatureWithDistinctParameterKind("test.dt.int00", Optional.empty(), "integer"))); + TypeSignature resolvedIntermediateType = + resolveIntermediateType(actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeGenericDistinctParametricType() + { + TypeSignature actualIntermediateType = new + TypeSignature("test", + TypeSignatureParameter.of(parseTypeSignature("K")), + TypeSignatureParameter.of(parseTypeSignature("V"))); + + TypeSignature expectedIntermediateType = new + TypeSignature("test", + TypeSignatureParameter.of(parseTypeSignature("array(double)")), + TypeSignatureParameter.of( + createTypeSignatureWithDistinctParameterKind("test.dt.int00", Optional.empty(), "bigint"))); + TypeSignature resolvedIntermediateType = + resolveIntermediateType(actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeNonGenericRowDistinctParametricType() + { + TypeSignature actualIntermediateType = new + TypeSignature("row", + TypeSignatureParameter.of(new NamedTypeSignature(Optional.empty(), parseTypeSignature("K"))), + TypeSignatureParameter.of( + new NamedTypeSignature(Optional.empty(), createTypeSignatureWithDistinctParameterKind("test.dt.int00", Optional.empty(), "integer")))); + + TypeSignature expectedIntermediateType = new + TypeSignature("row", + TypeSignatureParameter.of(new NamedTypeSignature(Optional.empty(), parseTypeSignature("array(double)"))), + TypeSignatureParameter.of( + new NamedTypeSignature(Optional.empty(), createTypeSignatureWithDistinctParameterKind("test.dt.int00", Optional.empty(), "integer")))); + TypeSignature resolvedIntermediateType = + resolveIntermediateType(actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + @Test + public void testResolveIntermediateTypeGenericRowDistinctParametricType() + { + TypeSignature actualIntermediateType = new + TypeSignature("row", + TypeSignatureParameter.of(new NamedTypeSignature(Optional.empty(), parseTypeSignature("K"))), + TypeSignatureParameter.of(new NamedTypeSignature(Optional.empty(), parseTypeSignature("V")))); + + TypeSignature expectedIntermediateType = new + TypeSignature("row", + TypeSignatureParameter.of(new NamedTypeSignature(Optional.empty(), parseTypeSignature("array(double)"))), + TypeSignatureParameter.of( + new NamedTypeSignature(Optional.empty(), createTypeSignatureWithDistinctParameterKind("test.dt.int00", Optional.empty(), "integer")))); + TypeSignature resolvedIntermediateType = + resolveIntermediateType(actualIntermediateType, + actualIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures(), + expectedIntermediateType.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(resolvedIntermediateType, expectedIntermediateType); + } + + private static TypeSignature createTypeSignatureWithDistinctParameterKind(String name, Optional parent, String baseType) + { + return new TypeSignature( + new DistinctTypeInfo( + QualifiedObjectName.valueOf(name), + parseTypeSignature(baseType), + parent.map(QualifiedObjectName::valueOf), + true)); + } +} diff --git a/presto-docs/src/main/sphinx/presto_cpp/properties.rst b/presto-docs/src/main/sphinx/presto_cpp/properties.rst index 5a704f622d5f8..e5270c9ab08d2 100644 --- a/presto-docs/src/main/sphinx/presto_cpp/properties.rst +++ b/presto-docs/src/main/sphinx/presto_cpp/properties.rst @@ -85,6 +85,14 @@ alphabetical order. This property provides function signatures for built-in aggregation functions which are compatible with Velox. +``presto.default-namespace`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** ``string`` +* **Default value:** ``presto.default`` + + Specifies the namespace prefix for native C++ functions. + Worker Properties ----------------- diff --git a/presto-function-namespace-managers-common/pom.xml b/presto-function-namespace-managers-common/pom.xml new file mode 100644 index 0000000000000..f793bf91c47b9 --- /dev/null +++ b/presto-function-namespace-managers-common/pom.xml @@ -0,0 +1,78 @@ + + + 4.0.0 + + + presto-root + com.facebook.presto + 0.292-SNAPSHOT + + + + ${project.parent.basedir} + + + presto-function-namespace-managers-common + + + com.facebook.presto + presto-spi + + + + com.google.guava + guava + + + + javax.inject + javax.inject + + + + com.facebook.airlift + configuration + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + + io.airlift + units + provided + + + + com.google.code.findbugs + jsr305 + + + + com.google.inject + guice + + + + com.facebook.presto + presto-common + provided + + + + + com.facebook.presto + presto-testng-services + test + + + + org.testng + testng + test + + + diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/AbstractSqlInvokedFunctionNamespaceManager.java b/presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/AbstractSqlInvokedFunctionNamespaceManager.java similarity index 97% rename from presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/AbstractSqlInvokedFunctionNamespaceManager.java rename to presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/AbstractSqlInvokedFunctionNamespaceManager.java index 613dedabe1cd7..b5d17f3c59844 100644 --- a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/AbstractSqlInvokedFunctionNamespaceManager.java +++ b/presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/AbstractSqlInvokedFunctionNamespaceManager.java @@ -18,6 +18,7 @@ import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.block.BlockEncodingSerde; import com.facebook.presto.common.function.SqlFunctionResult; +import com.facebook.presto.common.type.Type; import com.facebook.presto.common.type.TypeManager; import com.facebook.presto.common.type.UserDefinedType; import com.facebook.presto.functionNamespace.execution.SqlFunctionExecutors; @@ -192,7 +193,7 @@ public Optional getUserDefinedType(QualifiedObjectName typeName } @Override - public final FunctionHandle getFunctionHandle(Optional transactionHandle, Signature signature) + public FunctionHandle getFunctionHandle(Optional transactionHandle, Signature signature) { checkCatalog(signature.getName()); // This is the only assumption in this class that we're dealing with sql-invoked regular function. @@ -363,10 +364,13 @@ protected AggregationFunctionImplementation sqlInvokedFunctionToAggregationImple "Need aggregationMetadata to get aggregation function implementation"); AggregationFunctionMetadata aggregationMetadata = function.getAggregationMetadata().get(); + List parameters = function.getSignature().getArgumentTypes().stream().map( + (typeManager::getType)).collect(toImmutableList()); return new SqlInvokedAggregationFunctionImplementation( typeManager.getType(aggregationMetadata.getIntermediateType()), typeManager.getType(function.getSignature().getReturnType()), - aggregationMetadata.isOrderSensitive()); + aggregationMetadata.isOrderSensitive(), + parameters); default: throw new IllegalStateException(format("Unknown function implementation type: %s", implementationType)); } diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/InvalidFunctionHandleException.java b/presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/InvalidFunctionHandleException.java similarity index 100% rename from presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/InvalidFunctionHandleException.java rename to presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/InvalidFunctionHandleException.java diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/JsonBasedUdfFunctionMetadata.java b/presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/JsonBasedUdfFunctionMetadata.java similarity index 82% rename from presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/JsonBasedUdfFunctionMetadata.java rename to presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/JsonBasedUdfFunctionMetadata.java index 3353d91e73458..c9304fbef5d28 100644 --- a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/JsonBasedUdfFunctionMetadata.java +++ b/presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/JsonBasedUdfFunctionMetadata.java @@ -18,6 +18,7 @@ import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.RoutineCharacteristics; import com.facebook.presto.spi.function.SqlFunctionId; +import com.facebook.presto.spi.function.TypeVariableConstraint; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; @@ -63,6 +64,15 @@ public class JsonBasedUdfFunctionMetadata * Optional Aggregate-specific metadata (required for aggregation functions) */ private final Optional aggregateMetadata; + /** + * Marked to indicate whether it is a variable arity function. + * A variable arity function can have a variable number of arguments of the specified type. + */ + private final boolean variableArity; + /** + * Optional list of the typeVariableConstraints. + */ + private final Optional> typeVariableConstraints; private final Optional functionId; private final Optional version; @@ -73,16 +83,19 @@ public JsonBasedUdfFunctionMetadata( @JsonProperty("outputType") TypeSignature outputType, @JsonProperty("paramTypes") List paramTypes, @JsonProperty("schema") String schema, + @JsonProperty("variableArity") boolean variableArity, @JsonProperty("routineCharacteristics") RoutineCharacteristics routineCharacteristics, @JsonProperty("aggregateMetadata") Optional aggregateMetadata, @JsonProperty("functionId") Optional functionId, - @JsonProperty("version") Optional version) + @JsonProperty("version") Optional version, + @JsonProperty("typeVariableConstraints") Optional> typeVariableConstraints) { this.docString = requireNonNull(docString, "docString is null"); this.functionKind = requireNonNull(functionKind, "functionKind is null"); this.outputType = requireNonNull(outputType, "outputType is null"); this.paramTypes = ImmutableList.copyOf(requireNonNull(paramTypes, "paramTypes is null")); this.schema = requireNonNull(schema, "schema is null"); + this.variableArity = variableArity; this.routineCharacteristics = requireNonNull(routineCharacteristics, "routineCharacteristics is null"); this.aggregateMetadata = requireNonNull(aggregateMetadata, "aggregateMetadata is null"); checkArgument( @@ -90,6 +103,7 @@ public JsonBasedUdfFunctionMetadata( "aggregateMetadata must be present for aggregation functions and absent otherwise"); this.functionId = requireNonNull(functionId, "functionId is null"); this.version = requireNonNull(version, "version is null"); + this.typeVariableConstraints = requireNonNull(typeVariableConstraints, "typeVariableConstraints is null"); } @JsonProperty @@ -128,6 +142,12 @@ public String getSchema() return schema; } + @JsonProperty + public boolean getVariableArity() + { + return variableArity; + } + @JsonProperty public RoutineCharacteristics getRoutineCharacteristics() { @@ -151,4 +171,10 @@ public Optional getVersion() { return version; } + + @JsonProperty + public Optional> getTypeVariableConstraints() + { + return typeVariableConstraints; + } } diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/ServingCatalog.java b/presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/ServingCatalog.java similarity index 100% rename from presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/ServingCatalog.java rename to presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/ServingCatalog.java diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/SqlInvokedFunctionNamespaceManagerConfig.java b/presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/SqlInvokedFunctionNamespaceManagerConfig.java similarity index 100% rename from presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/SqlInvokedFunctionNamespaceManagerConfig.java rename to presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/SqlInvokedFunctionNamespaceManagerConfig.java diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/UdfFunctionSignatureMap.java b/presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/UdfFunctionSignatureMap.java similarity index 100% rename from presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/UdfFunctionSignatureMap.java rename to presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/UdfFunctionSignatureMap.java diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/UuidFunctionNamespaceTransactionHandle.java b/presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/UuidFunctionNamespaceTransactionHandle.java similarity index 100% rename from presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/UuidFunctionNamespaceTransactionHandle.java rename to presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/UuidFunctionNamespaceTransactionHandle.java diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/execution/NoopSqlFunctionExecutionModule.java b/presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/execution/NoopSqlFunctionExecutionModule.java similarity index 100% rename from presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/execution/NoopSqlFunctionExecutionModule.java rename to presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/execution/NoopSqlFunctionExecutionModule.java diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/execution/NoopSqlFunctionExecutor.java b/presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/execution/NoopSqlFunctionExecutor.java similarity index 100% rename from presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/execution/NoopSqlFunctionExecutor.java rename to presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/execution/NoopSqlFunctionExecutor.java diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/execution/NoopSqlFunctionExecutorsModule.java b/presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/execution/NoopSqlFunctionExecutorsModule.java similarity index 100% rename from presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/execution/NoopSqlFunctionExecutorsModule.java rename to presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/execution/NoopSqlFunctionExecutorsModule.java diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/execution/SqlFunctionExecutionModule.java b/presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/execution/SqlFunctionExecutionModule.java similarity index 100% rename from presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/execution/SqlFunctionExecutionModule.java rename to presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/execution/SqlFunctionExecutionModule.java diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/execution/SqlFunctionExecutors.java b/presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/execution/SqlFunctionExecutors.java similarity index 100% rename from presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/execution/SqlFunctionExecutors.java rename to presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/execution/SqlFunctionExecutors.java diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/execution/SqlFunctionLanguageConfig.java b/presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/execution/SqlFunctionLanguageConfig.java similarity index 100% rename from presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/execution/SqlFunctionLanguageConfig.java rename to presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/execution/SqlFunctionLanguageConfig.java diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/testing/InMemoryFunctionNamespaceManager.java b/presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/testing/InMemoryFunctionNamespaceManager.java similarity index 100% rename from presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/testing/InMemoryFunctionNamespaceManager.java rename to presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/testing/InMemoryFunctionNamespaceManager.java diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/testing/SqlInvokedFunctionTestUtils.java b/presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/testing/SqlInvokedFunctionTestUtils.java similarity index 100% rename from presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/testing/SqlInvokedFunctionTestUtils.java rename to presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/testing/SqlInvokedFunctionTestUtils.java diff --git a/presto-function-namespace-managers/src/test/java/com/facebook/presto/functionNamespace/TestSqlInvokedFunctionNamespaceManager.java b/presto-function-namespace-managers-common/src/test/java/com/facebook/presto/functionNamespace/TestSqlInvokedFunctionNamespaceManager.java similarity index 100% rename from presto-function-namespace-managers/src/test/java/com/facebook/presto/functionNamespace/TestSqlInvokedFunctionNamespaceManager.java rename to presto-function-namespace-managers-common/src/test/java/com/facebook/presto/functionNamespace/TestSqlInvokedFunctionNamespaceManager.java diff --git a/presto-function-namespace-managers/src/test/java/com/facebook/presto/functionNamespace/TestSqlInvokedFunctionNamespaceManagerConfig.java b/presto-function-namespace-managers-common/src/test/java/com/facebook/presto/functionNamespace/TestSqlInvokedFunctionNamespaceManagerConfig.java similarity index 100% rename from presto-function-namespace-managers/src/test/java/com/facebook/presto/functionNamespace/TestSqlInvokedFunctionNamespaceManagerConfig.java rename to presto-function-namespace-managers-common/src/test/java/com/facebook/presto/functionNamespace/TestSqlInvokedFunctionNamespaceManagerConfig.java diff --git a/presto-function-namespace-managers/src/test/java/com/facebook/presto/functionNamespace/execution/TestSqlFunctionLanguageConfig.java b/presto-function-namespace-managers-common/src/test/java/com/facebook/presto/functionNamespace/execution/TestSqlFunctionLanguageConfig.java similarity index 100% rename from presto-function-namespace-managers/src/test/java/com/facebook/presto/functionNamespace/execution/TestSqlFunctionLanguageConfig.java rename to presto-function-namespace-managers-common/src/test/java/com/facebook/presto/functionNamespace/execution/TestSqlFunctionLanguageConfig.java diff --git a/presto-function-namespace-managers/pom.xml b/presto-function-namespace-managers/pom.xml index 0d7f770079b04..c5e50a70e1bb0 100644 --- a/presto-function-namespace-managers/pom.xml +++ b/presto-function-namespace-managers/pom.xml @@ -85,11 +85,6 @@ provided - - com.google.code.findbugs - jsr305 - - com.google.guava guava @@ -217,5 +212,13 @@ javax.ws.rs-api test + + + com.facebook.presto + presto-function-namespace-managers-common + ${project.version} + compile + + diff --git a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionNamespaceManager.java b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionNamespaceManager.java index c9cd27f6318f6..0770a2d5e7dec 100644 --- a/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionNamespaceManager.java +++ b/presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionNamespaceManager.java @@ -143,6 +143,7 @@ private SqlInvokedFunction createSqlInvokedFunction(String functionName, JsonBas jsonBasedUdfFunctionMetaData.getDocString(), jsonBasedUdfFunctionMetaData.getRoutineCharacteristics(), "", + jsonBasedUdfFunctionMetaData.getVariableArity(), functionVersion, jsonBasedUdfFunctionMetaData.getFunctionKind(), functionId, diff --git a/presto-function-namespace-managers/src/test/java/com/facebook/presto/functionNamespace/TestRestBasedFunctionNamespaceManager.java b/presto-function-namespace-managers/src/test/java/com/facebook/presto/functionNamespace/TestRestBasedFunctionNamespaceManager.java index 5eabad9db3d5b..7045f9f165b3c 100644 --- a/presto-function-namespace-managers/src/test/java/com/facebook/presto/functionNamespace/TestRestBasedFunctionNamespaceManager.java +++ b/presto-function-namespace-managers/src/test/java/com/facebook/presto/functionNamespace/TestRestBasedFunctionNamespaceManager.java @@ -58,6 +58,7 @@ import java.util.Optional; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; +import static java.util.Collections.emptyList; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; @@ -140,20 +141,24 @@ public static Map> createUdfSignature new TypeSignature("integer"), Collections.singletonList(new TypeSignature("integer")), "default", + false, new RoutineCharacteristics(RoutineCharacteristics.Language.CPP, RoutineCharacteristics.Determinism.DETERMINISTIC, RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT), Optional.empty(), Optional.of(new SqlFunctionId(QualifiedObjectName.valueOf("unittest.default.square"), ImmutableList.of(parseTypeSignature("integer")))), - Optional.of("1"))); + Optional.of("1"), + Optional.of(emptyList()))); squareFunctions.add(new JsonBasedUdfFunctionMetadata( "square a double", FunctionKind.SCALAR, new TypeSignature("double"), Collections.singletonList(new TypeSignature("double")), "test_schema", + false, new RoutineCharacteristics(RoutineCharacteristics.Language.CPP, RoutineCharacteristics.Determinism.DETERMINISTIC, RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT), Optional.empty(), Optional.of(new SqlFunctionId(QualifiedObjectName.valueOf("unittest.test_schema.square"), ImmutableList.of(parseTypeSignature("double")))), - Optional.of("1"))); + Optional.of("1"), + Optional.of(emptyList()))); udfSignatureMap.put("square", squareFunctions); // array_function_1 @@ -164,30 +169,36 @@ public static Map> createUdfSignature parseTypeSignature("ARRAY>"), Arrays.asList(parseTypeSignature("ARRAY>"), parseTypeSignature("ARRAY>")), "default", + false, new RoutineCharacteristics(RoutineCharacteristics.Language.CPP, RoutineCharacteristics.Determinism.DETERMINISTIC, RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT), Optional.empty(), Optional.of(new SqlFunctionId(QualifiedObjectName.valueOf("unittest.default.array_function_1"), ImmutableList.of(parseTypeSignature("ARRAY>"), parseTypeSignature("ARRAY>")))), - Optional.of("1"))); + Optional.of("1"), + Optional.of(emptyList()))); arrayFunction1.add(new JsonBasedUdfFunctionMetadata( "combines two float arrays into one", FunctionKind.SCALAR, parseTypeSignature("ARRAY>"), Arrays.asList(parseTypeSignature("ARRAY>"), parseTypeSignature("ARRAY>")), "test_schema", + false, new RoutineCharacteristics(RoutineCharacteristics.Language.CPP, RoutineCharacteristics.Determinism.DETERMINISTIC, RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT), Optional.empty(), Optional.of(new SqlFunctionId(QualifiedObjectName.valueOf("unittest.test_schema.array_function_1"), ImmutableList.of(parseTypeSignature("ARRAY>"), parseTypeSignature("ARRAY>")))), - Optional.of("1"))); + Optional.of("1"), + Optional.of(emptyList()))); arrayFunction1.add(new JsonBasedUdfFunctionMetadata( "combines two double arrays into one", FunctionKind.SCALAR, parseTypeSignature("ARRAY"), Arrays.asList(parseTypeSignature("ARRAY"), TypeSignature.parseTypeSignature("ARRAY")), "test_schema", + false, new RoutineCharacteristics(RoutineCharacteristics.Language.CPP, RoutineCharacteristics.Determinism.DETERMINISTIC, RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT), Optional.empty(), Optional.of(new SqlFunctionId(QualifiedObjectName.valueOf("unittest.test_schema.array_function_1"), ImmutableList.of(parseTypeSignature("ARRAY"), parseTypeSignature("ARRAY")))), - Optional.of("1"))); + Optional.of("1"), + Optional.of(emptyList()))); udfSignatureMap.put("array_function_1", arrayFunction1); // array_function_2 @@ -198,20 +209,24 @@ public static Map> createUdfSignature TypeSignature.parseTypeSignature("ARRAY>"), Arrays.asList(TypeSignature.parseTypeSignature("ARRAY>"), TypeSignature.parseTypeSignature("ARRAY")), "default", + false, new RoutineCharacteristics(RoutineCharacteristics.Language.CPP, RoutineCharacteristics.Determinism.DETERMINISTIC, RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT), Optional.empty(), Optional.of(new SqlFunctionId(QualifiedObjectName.valueOf("unittest.default.array_function_2"), ImmutableList.of(parseTypeSignature("ARRAY>"), parseTypeSignature("ARRAY")))), - Optional.of("1"))); + Optional.of("1"), + Optional.of(emptyList()))); arrayFunction2.add(new JsonBasedUdfFunctionMetadata( "transforms inputs into the output", FunctionKind.SCALAR, TypeSignature.parseTypeSignature("ARRAY>"), Arrays.asList(TypeSignature.parseTypeSignature("ARRAY>"), TypeSignature.parseTypeSignature("ARRAY>"), TypeSignature.parseTypeSignature("ARRAY")), "test_schema", + false, new RoutineCharacteristics(RoutineCharacteristics.Language.CPP, RoutineCharacteristics.Determinism.DETERMINISTIC, RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT), Optional.empty(), Optional.of(new SqlFunctionId(QualifiedObjectName.valueOf("unittest.test_schema.array_function_2"), ImmutableList.of(parseTypeSignature("ARRAY>"), parseTypeSignature("ARRAY>"), parseTypeSignature("ARRAY")))), - Optional.of("1"))); + Optional.of("1"), + Optional.of(emptyList()))); udfSignatureMap.put("array_function_2", arrayFunction2); return udfSignatureMap; @@ -229,20 +244,24 @@ public static Map> createUpdatedUdfSi new TypeSignature("integer"), Collections.singletonList(new TypeSignature("integer")), "default", + false, new RoutineCharacteristics(RoutineCharacteristics.Language.CPP, RoutineCharacteristics.Determinism.DETERMINISTIC, RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT), Optional.empty(), Optional.of(new SqlFunctionId(QualifiedObjectName.valueOf("unittest.default.square"), ImmutableList.of(parseTypeSignature("integer")))), - Optional.of("1"))); + Optional.of("1"), + Optional.of(emptyList()))); squareFunctions.add(new JsonBasedUdfFunctionMetadata( "square a double", FunctionKind.SCALAR, new TypeSignature("double"), Collections.singletonList(new TypeSignature("double")), "test_schema", + false, new RoutineCharacteristics(RoutineCharacteristics.Language.CPP, RoutineCharacteristics.Determinism.DETERMINISTIC, RoutineCharacteristics.NullCallClause.CALLED_ON_NULL_INPUT), Optional.empty(), Optional.of(new SqlFunctionId(QualifiedObjectName.valueOf("unittest.test_schema.square"), ImmutableList.of(parseTypeSignature("double")))), - Optional.of("1"))); + Optional.of("1"), + Optional.of(emptyList()))); udfSignatureMap.put("square", squareFunctions); return udfSignatureMap; diff --git a/presto-function-server/pom.xml b/presto-function-server/pom.xml index 710de31fa3e8a..4a652e13e25bd 100644 --- a/presto-function-server/pom.xml +++ b/presto-function-server/pom.xml @@ -129,6 +129,12 @@ aether-api + + com.facebook.presto + presto-function-namespace-managers-common + ${project.version} + + com.facebook.presto presto-tests diff --git a/presto-function-server/src/main/java/com/facebook/presto/server/FunctionResource.java b/presto-function-server/src/main/java/com/facebook/presto/server/FunctionResource.java index e67a685b00fca..2ec8b66b38416 100644 --- a/presto-function-server/src/main/java/com/facebook/presto/server/FunctionResource.java +++ b/presto-function-server/src/main/java/com/facebook/presto/server/FunctionResource.java @@ -129,6 +129,7 @@ private static JsonBasedUdfFunctionMetadata sqlFunctionToMetadata(SqlFunction fu function.getSignature().getReturnType(), function.getSignature().getArgumentTypes(), function.getSignature().getName().getSchemaName(), + function.getSignature().isVariableArity(), new RoutineCharacteristics( JAVA, function.isDeterministic() ? DETERMINISTIC : NOT_DETERMINISTIC, @@ -138,7 +139,8 @@ private static JsonBasedUdfFunctionMetadata sqlFunctionToMetadata(SqlFunction fu new SqlFunctionId( function.getSignature().getName(), function.getSignature().getArgumentTypes())), - Optional.of("1")); + Optional.of("1"), + Optional.of(function.getSignature().getTypeVariableConstraints())); } @GET diff --git a/presto-grpc-api/pom.xml b/presto-grpc-api/pom.xml index e603284b2c73c..f9b99119b0a31 100644 --- a/presto-grpc-api/pom.xml +++ b/presto-grpc-api/pom.xml @@ -80,7 +80,8 @@ com.facebook.presto - presto-function-namespace-managers + presto-function-namespace-managers-common + ${project.version} diff --git a/presto-hive-function-namespace/src/main/java/com/facebook/presto/hive/functions/HiveFunctionHandle.java b/presto-hive-function-namespace/src/main/java/com/facebook/presto/hive/functions/HiveFunctionHandle.java index 5a9cca018115f..1e8159af10528 100644 --- a/presto-hive-function-namespace/src/main/java/com/facebook/presto/hive/functions/HiveFunctionHandle.java +++ b/presto-hive-function-namespace/src/main/java/com/facebook/presto/hive/functions/HiveFunctionHandle.java @@ -15,12 +15,14 @@ package com.facebook.presto.hive.functions; import com.facebook.presto.common.CatalogSchemaName; +import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.Signature; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; import java.util.Objects; import static java.util.Objects.requireNonNull; @@ -54,6 +56,12 @@ public FunctionKind getKind() return signature.getKind(); } + @Override + public List getArgumentTypes() + { + return signature.getArgumentTypes(); + } + @JsonProperty public Signature getSignature() { diff --git a/presto-hive-function-namespace/src/test/java/com/facebook/presto/hive/functions/HiveFunctionsTestUtils.java b/presto-hive-function-namespace/src/test/java/com/facebook/presto/hive/functions/HiveFunctionsTestUtils.java index 5bdcae9839969..f69fbf53c4349 100644 --- a/presto-hive-function-namespace/src/test/java/com/facebook/presto/hive/functions/HiveFunctionsTestUtils.java +++ b/presto-hive-function-namespace/src/test/java/com/facebook/presto/hive/functions/HiveFunctionsTestUtils.java @@ -37,7 +37,8 @@ public static TestingPrestoServer createTestingPrestoServer() functionAndTypeManager.loadFunctionNamespaceManager( "hive-functions", "hive", - getNamespaceManagerCreationProperties()); + getNamespaceManagerCreationProperties(), + server.getPluginNodeManager()); server.refreshNodes(); return server; } diff --git a/presto-hive-function-namespace/src/test/java/com/facebook/presto/hive/functions/TestHiveAggregationFunctions.java b/presto-hive-function-namespace/src/test/java/com/facebook/presto/hive/functions/TestHiveAggregationFunctions.java index 03294c4c98d2d..981477c77229e 100644 --- a/presto-hive-function-namespace/src/test/java/com/facebook/presto/hive/functions/TestHiveAggregationFunctions.java +++ b/presto-hive-function-namespace/src/test/java/com/facebook/presto/hive/functions/TestHiveAggregationFunctions.java @@ -149,7 +149,8 @@ private static TestingPrestoServer createServer() functionAndTypeManager.loadFunctionNamespaceManager( "hive-functions", "hive", - Collections.emptyMap()); + Collections.emptyMap(), + server.getPluginNodeManager()); server.refreshNodes(); return server; } diff --git a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergMetadataOptimizer.java b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergMetadataOptimizer.java index 3e35e1d6afa4a..3622947acf497 100644 --- a/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergMetadataOptimizer.java +++ b/presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergMetadataOptimizer.java @@ -13,8 +13,6 @@ */ package com.facebook.presto.iceberg.optimizer; -import com.facebook.presto.common.CatalogSchemaName; -import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.predicate.NullableValue; import com.facebook.presto.common.predicate.TupleDomain; import com.facebook.presto.common.predicate.TupleDomain.ColumnDomain; @@ -58,7 +56,6 @@ import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; @@ -67,7 +64,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; +import java.util.function.Predicate; import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; import static com.facebook.presto.iceberg.IcebergSessionProperties.getRowsForMetadataOptimizationThreshold; @@ -81,16 +78,10 @@ public class IcebergMetadataOptimizer implements ConnectorPlanOptimizer { - public static final CatalogSchemaName DEFAULT_NAMESPACE = new CatalogSchemaName("presto", "default"); - private static final Set ALLOWED_FUNCTIONS = ImmutableSet.of( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "max"), - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "min"), - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "approx_distinct")); - // Min/Max could be folded into LEAST/GREATEST - private static final Map AGGREGATION_SCALAR_MAPPING = ImmutableMap.of( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "max"), QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "greatest"), - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "min"), QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "least")); + private static final Map AGGREGATION_SCALAR_MAPPING = ImmutableMap.of( + "max", "greatest", + "min", "least"); private final FunctionMetadataManager functionMetadataManager; private final TypeManager typeManager; @@ -137,6 +128,7 @@ private static class Optimizer private final RowExpressionService rowExpressionService; private final StandardFunctionResolution functionResolution; private final int rowsForMetadataOptimizationThreshold; + private final List> allowedFunctionsPredicates; private Optimizer(ConnectorSession connectorSession, PlanNodeIdAllocator idAllocator, @@ -156,6 +148,10 @@ private Optimizer(ConnectorSession connectorSession, this.functionResolution = functionResolution; this.typeManager = typeManager; this.rowsForMetadataOptimizationThreshold = rowsForMetadataOptimizationThreshold; + this.allowedFunctionsPredicates = ImmutableList.of( + functionResolution::isMaxFunction, + functionResolution::isMinFunction, + functionResolution::isApproximateCountDistinctFunction); } @Override @@ -163,8 +159,8 @@ public PlanNode visitAggregation(AggregationNode node, RewriteContext cont { // supported functions are only MIN/MAX/APPROX_DISTINCT or distinct aggregates for (Aggregation aggregation : node.getAggregations().values()) { - QualifiedObjectName functionName = functionMetadataManager.getFunctionMetadata(aggregation.getFunctionHandle()).getName(); - if (!ALLOWED_FUNCTIONS.contains(functionName) && !aggregation.isDistinct()) { + if (allowedFunctionsPredicates.stream().noneMatch( + pred -> pred.test(aggregation.getFunctionHandle())) && !aggregation.isDistinct()) { return context.defaultRewrite(node); } } @@ -270,7 +266,7 @@ private boolean isReducible(AggregationNode node, List 1 || !inputs.containsAll(aggregation.getCall().getArguments())) { return false; @@ -340,7 +336,7 @@ private RowExpression evaluateMinMax(FunctionMetadata aggregationFunctionMetadat return new ConstantExpression(Optional.empty(), null, returnType); } - String scalarFunctionName = AGGREGATION_SCALAR_MAPPING.get(aggregationFunctionMetadata.getName()).getObjectName(); + String scalarFunctionName = AGGREGATION_SCALAR_MAPPING.get(aggregationFunctionMetadata.getName().getObjectName()); while (arguments.size() > 1) { List reducedArguments = new ArrayList<>(); // We fold for every 100 values because GREATEST/LEAST has argument count limit diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 09527a6277c9a..3e5621ba35663 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -38,6 +38,13 @@ test + + com.facebook.presto + presto-function-namespace-managers-common + ${project.version} + test + + com.facebook.presto presto-tests diff --git a/presto-main/pom.xml b/presto-main/pom.xml index a194bbee8a498..19180edabdf03 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -469,6 +469,13 @@ test + + com.facebook.presto + presto-function-namespace-managers-common + ${project.version} + test + + com.facebook.presto presto-plugin-toolkit diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunctionHandle.java b/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunctionHandle.java index e554e7fe0f7c7..40de75c838169 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunctionHandle.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunctionHandle.java @@ -14,12 +14,14 @@ package com.facebook.presto.metadata; import com.facebook.presto.common.CatalogSchemaName; +import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.function.Signature; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; import java.util.Objects; import static java.util.Objects.requireNonNull; @@ -54,6 +56,12 @@ public FunctionKind getKind() return signature.getKind(); } + @Override + public List getArgumentTypes() + { + return signature.getArgumentTypes(); + } + @Override public CatalogSchemaName getCatalogSchemaName() { diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java index ae76fd532db64..c419822e91405 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java @@ -236,7 +236,6 @@ import com.facebook.presto.spi.function.SqlInvokedFunction; import com.facebook.presto.spi.function.SqlInvokedScalarFunctionImplementation; import com.facebook.presto.sql.analyzer.FunctionsConfig; -import com.facebook.presto.sql.analyzer.TypeSignatureProvider; import com.facebook.presto.type.BigintOperators; import com.facebook.presto.type.BooleanOperators; import com.facebook.presto.type.CharOperators; @@ -345,7 +344,6 @@ import static com.facebook.presto.geospatial.SphericalGeographyType.SPHERICAL_GEOGRAPHY; import static com.facebook.presto.geospatial.type.BingTileType.BING_TILE; import static com.facebook.presto.geospatial.type.GeometryType.GEOMETRY; -import static com.facebook.presto.metadata.SignatureBinder.applyBoundVariables; import static com.facebook.presto.operator.aggregation.AlternativeArbitraryAggregationFunction.ALTERNATIVE_ANY_VALUE_AGGREGATION; import static com.facebook.presto.operator.aggregation.AlternativeArbitraryAggregationFunction.ALTERNATIVE_ARBITRARY_AGGREGATION; import static com.facebook.presto.operator.aggregation.AlternativeMaxAggregationFunction.ALTERNATIVE_MAX; @@ -461,7 +459,6 @@ import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.spi.function.FunctionKind.WINDOW; import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; -import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypeSignatures; import static com.facebook.presto.sql.planner.LiteralEncoder.MAGIC_LITERAL_FUNCTION_PREFIX; import static com.facebook.presto.type.ArrayParametricType.ARRAY; import static com.facebook.presto.type.CodePointsType.CODE_POINTS; @@ -521,7 +518,6 @@ import static com.facebook.presto.type.Re2JRegexpType.RE2J_REGEXP; import static com.facebook.presto.type.RowParametricType.ROW; import static com.facebook.presto.type.SfmSketchType.SFM_SKETCH; -import static com.facebook.presto.type.TypeUtils.resolveTypes; import static com.facebook.presto.type.khyperloglog.KHyperLogLogType.K_HYPER_LOG_LOG; import static com.facebook.presto.type.setdigest.SetDigestType.SET_DIGEST; import static com.google.common.base.Preconditions.checkArgument; @@ -537,7 +533,7 @@ public class BuiltInTypeAndFunctionNamespaceManager implements FunctionNamespaceManager { - public static final CatalogSchemaName DEFAULT_NAMESPACE = new CatalogSchemaName("presto", "default"); + public static final CatalogSchemaName JAVA_BUILTIN_NAMESPACE = new CatalogSchemaName("presto", "default"); public static final String ID = "builtin"; private final FunctionAndTypeManager functionAndTypeManager; @@ -1343,47 +1339,11 @@ private SpecializedFunctionKey getSpecializedFunctionKey(Signature signature) private SpecializedFunctionKey doGetSpecializedFunctionKey(Signature signature) { - Iterable candidates = getFunctions(null, signature.getName()); - // search for exact match - Type returnType = functionAndTypeManager.getType(signature.getReturnType()); - List argumentTypeSignatureProviders = fromTypeSignatures(signature.getArgumentTypes()); - for (SqlFunction candidate : candidates) { - Optional boundVariables = new SignatureBinder(functionAndTypeManager, candidate.getSignature(), false) - .bindVariables(argumentTypeSignatureProviders, returnType); - if (boundVariables.isPresent()) { - return new SpecializedFunctionKey(candidate, boundVariables.get(), argumentTypeSignatureProviders.size()); - } - } - - // TODO: hack because there could be "type only" coercions (which aren't necessarily included as implicit casts), - // so do a second pass allowing "type only" coercions - List argumentTypes = resolveTypes(signature.getArgumentTypes(), functionAndTypeManager); - for (SqlFunction candidate : candidates) { - SignatureBinder binder = new SignatureBinder(functionAndTypeManager, candidate.getSignature(), true); - Optional boundVariables = binder.bindVariables(argumentTypeSignatureProviders, returnType); - if (!boundVariables.isPresent()) { - continue; - } - Signature boundSignature = applyBoundVariables(candidate.getSignature(), boundVariables.get(), argumentTypes.size()); - - if (!functionAndTypeManager.isTypeOnlyCoercion(functionAndTypeManager.getType(boundSignature.getReturnType()), returnType)) { - continue; - } - boolean nonTypeOnlyCoercion = false; - for (int i = 0; i < argumentTypes.size(); i++) { - Type expectedType = functionAndTypeManager.getType(boundSignature.getArgumentTypes().get(i)); - if (!functionAndTypeManager.isTypeOnlyCoercion(argumentTypes.get(i), expectedType)) { - nonTypeOnlyCoercion = true; - break; - } - } - if (nonTypeOnlyCoercion) { - continue; - } - - return new SpecializedFunctionKey(candidate, boundVariables.get(), argumentTypes.size()); - } + return functionAndTypeManager.getSpecializedFunctionKey(signature); + } + public SpecializedFunctionKey doGetSpecializedFunctionKeyForMagicLiteralFunctions(Signature signature, FunctionAndTypeManager functionAndTypeManager) + { // TODO: this is a hack and should be removed if (signature.getNameSuffix().startsWith(MAGIC_LITERAL_FUNCTION_PREFIX)) { List parameterTypes = signature.getArgumentTypes(); @@ -1406,7 +1366,6 @@ private SpecializedFunctionKey doGetSpecializedFunctionKey(Signature signature) .build(), 1); } - throw new PrestoException(FUNCTION_IMPLEMENTATION_MISSING, format("%s not found", signature)); } @@ -1546,7 +1505,7 @@ private static class MagicLiteralFunction MagicLiteralFunction(BlockEncodingSerde blockEncodingSerde) { - super(new Signature(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, MAGIC_LITERAL_FUNCTION_PREFIX), SCALAR, TypeSignature.parseTypeSignature("R"), TypeSignature.parseTypeSignature("T"))); + super(new Signature(QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, MAGIC_LITERAL_FUNCTION_PREFIX), SCALAR, TypeSignature.parseTypeSignature("R"), TypeSignature.parseTypeSignature("T"))); this.blockEncodingSerde = requireNonNull(blockEncodingSerde, "blockEncodingSerde is null"); } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/CastType.java b/presto-main/src/main/java/com/facebook/presto/metadata/CastType.java index e0cdea53cd494..395cbf76ba4fe 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/CastType.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/CastType.java @@ -16,7 +16,7 @@ import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.function.OperatorType; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.JsonStringToArrayCast.JSON_STRING_TO_ARRAY_NAME; import static com.facebook.presto.operator.scalar.JsonStringToMapCast.JSON_STRING_TO_MAP_NAME; import static com.facebook.presto.operator.scalar.JsonStringToRowCast.JSON_STRING_TO_ROW_NAME; @@ -27,10 +27,10 @@ public enum CastType { CAST(OperatorType.CAST.getFunctionName(), true), SATURATED_FLOOR_CAST(OperatorType.SATURATED_FLOOR_CAST.getFunctionName(), true), - TRY_CAST(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, TRY_CAST_NAME), false), - JSON_TO_ARRAY_CAST(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, JSON_STRING_TO_ARRAY_NAME), false), - JSON_TO_MAP_CAST(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, JSON_STRING_TO_MAP_NAME), false), - JSON_TO_ROW_CAST(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, JSON_STRING_TO_ROW_NAME), false); + TRY_CAST(QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, TRY_CAST_NAME), false), + JSON_TO_ARRAY_CAST(QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, JSON_STRING_TO_ARRAY_NAME), false), + JSON_TO_MAP_CAST(QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, JSON_STRING_TO_MAP_NAME), false), + JSON_TO_ROW_CAST(QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, JSON_STRING_TO_ROW_NAME), false); private final QualifiedObjectName castName; private final boolean isOperatorType; diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionAndTypeManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionAndTypeManager.java index 8c039bd57b76c..456bdef5dba80 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionAndTypeManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionAndTypeManager.java @@ -33,6 +33,7 @@ import com.facebook.presto.common.type.TypeWithName; import com.facebook.presto.common.type.UserDefinedType; import com.facebook.presto.operator.window.WindowFunctionSupplier; +import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.function.AggregationFunctionImplementation; import com.facebook.presto.spi.function.AlterRoutineCharacteristics; @@ -49,6 +50,7 @@ import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.function.SqlFunction; import com.facebook.presto.spi.function.SqlFunctionId; +import com.facebook.presto.spi.function.SqlFunctionSupplier; import com.facebook.presto.spi.function.SqlInvokedFunction; import com.facebook.presto.sql.analyzer.FeaturesConfig; import com.facebook.presto.sql.analyzer.FunctionAndTypeResolver; @@ -80,14 +82,16 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; import static com.facebook.presto.SystemSessionProperties.isExperimentalFunctionsEnabled; import static com.facebook.presto.SystemSessionProperties.isListBuiltInFunctionsOnly; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.metadata.CastType.toOperatorType; import static com.facebook.presto.metadata.FunctionSignatureMatcher.constructFunctionNotFoundErrorMessage; import static com.facebook.presto.metadata.SessionFunctionHandle.SESSION_NAMESPACE; +import static com.facebook.presto.metadata.SignatureBinder.applyBoundVariables; import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_MISSING; import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_NOT_FOUND; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_USER_ERROR; @@ -98,6 +102,7 @@ import static com.facebook.presto.sql.planner.LiteralEncoder.MAGIC_LITERAL_FUNCTION_PREFIX; import static com.facebook.presto.sql.planner.LiteralEncoder.getMagicLiteralFunctionSignature; import static com.facebook.presto.transaction.InMemoryTransactionManager.createTestTransactionManager; +import static com.facebook.presto.type.TypeUtils.resolveTypes; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; @@ -117,6 +122,7 @@ public class FunctionAndTypeManager implements FunctionMetadataManager, TypeManager { + private static final Pattern DEFAULT_NAMESPACE_PREFIX_PATTERN = Pattern.compile("[a-z]+\\.[a-z]+"); private final TransactionManager transactionManager; private final BlockEncodingSerde blockEncodingSerde; private final BuiltInTypeAndFunctionNamespaceManager builtInTypeAndFunctionNamespaceManager; @@ -129,6 +135,7 @@ public class FunctionAndTypeManager private final LoadingCache functionCache; private final CacheStatsMBean cacheStatsMBean; private final boolean nativeExecution; + private final CatalogSchemaName defaultNamespace; @Inject public FunctionAndTypeManager( @@ -142,11 +149,11 @@ public FunctionAndTypeManager( this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); this.blockEncodingSerde = requireNonNull(blockEncodingSerde, "blockEncodingSerde is null"); this.builtInTypeAndFunctionNamespaceManager = new BuiltInTypeAndFunctionNamespaceManager(blockEncodingSerde, functionsConfig, types, this); - this.functionNamespaceManagers.put(DEFAULT_NAMESPACE.getCatalogName(), builtInTypeAndFunctionNamespaceManager); + this.functionNamespaceManagers.put(JAVA_BUILTIN_NAMESPACE.getCatalogName(), builtInTypeAndFunctionNamespaceManager); this.functionInvokerProvider = new FunctionInvokerProvider(this); this.handleResolver = requireNonNull(handleResolver, "handleResolver is null"); // TODO: Provide a more encapsulated way for TransactionManager to register FunctionNamespaceManager - transactionManager.registerFunctionNamespaceManager(DEFAULT_NAMESPACE.getCatalogName(), builtInTypeAndFunctionNamespaceManager); + transactionManager.registerFunctionNamespaceManager(JAVA_BUILTIN_NAMESPACE.getCatalogName(), builtInTypeAndFunctionNamespaceManager); this.functionCache = CacheBuilder.newBuilder() .recordStats() .maximumSize(1000) @@ -156,6 +163,7 @@ public FunctionAndTypeManager( this.functionSignatureMatcher = new FunctionSignatureMatcher(this); this.typeCoercer = new TypeCoercer(functionsConfig, this); this.nativeExecution = featuresConfig.isNativeExecutionEnabled(); + this.defaultNamespace = configureDefaultNamespace(functionsConfig.getDefaultNamespacePrefix()); } public static FunctionAndTypeManager createTestFunctionAndTypeManager() @@ -220,6 +228,12 @@ public FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle) return FunctionAndTypeManager.this.getFunctionMetadata(functionHandle); } + @Override + public SqlFunctionSupplier getSpecializedFunctionKey(Signature signature) + { + return FunctionAndTypeManager.this.getSpecializedFunctionKey(signature); + } + @Override public Collection listBuiltInFunctions() { @@ -246,8 +260,11 @@ public FunctionHandle lookupCast(String castType, Type fromType, Type toType) public QualifiedObjectName qualifyObjectName(QualifiedName name) { + if (name.getSuffix().startsWith("$internal")) { + return QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, name.getSuffix()); + } if (!name.getPrefix().isPresent()) { - return QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, name.getSuffix()); + return QualifiedObjectName.valueOf(defaultNamespace, name.getSuffix()); } if (name.getOriginalParts().size() != 3) { throw new PrestoException(FUNCTION_NOT_FOUND, format("Functions that are not temporary or builtin must be referenced by 'catalog.schema.function_name', found: %s", name)); @@ -267,12 +284,13 @@ public CacheStatsMBean getFunctionResolutionCacheStats() public void loadFunctionNamespaceManager( String functionNamespaceManagerName, String catalogName, - Map properties) + Map properties, + NodeManager nodeManager) { requireNonNull(functionNamespaceManagerName, "functionNamespaceManagerName is null"); FunctionNamespaceManagerFactory factory = functionNamespaceManagerFactories.get(functionNamespaceManagerName); checkState(factory != null, "No factory for function namespace manager %s", functionNamespaceManagerName); - FunctionNamespaceManager functionNamespaceManager = factory.create(catalogName, properties, new FunctionNamespaceManagerContext(this)); + FunctionNamespaceManager functionNamespaceManager = factory.create(catalogName, properties, new FunctionNamespaceManagerContext(this, nodeManager, this)); functionNamespaceManager.setBlockEncodingSerde(blockEncodingSerde); transactionManager.registerFunctionNamespaceManager(catalogName, functionNamespaceManager); @@ -361,15 +379,20 @@ public void registerBuiltInFunctions(List functions) public List listFunctions(Session session, Optional likePattern, Optional escape) { ImmutableList.Builder functions = new ImmutableList.Builder<>(); - if (!isListBuiltInFunctionsOnly(session)) { + if (isListBuiltInFunctionsOnly(session)) { + if (!functionNamespaceManagers.containsKey(defaultNamespace.getCatalogName())) { + throw new PrestoException(GENERIC_USER_ERROR, format("Function namespace not found for catalog: %s", defaultNamespace.getCatalogName())); + } + functions.addAll(functionNamespaceManagers.get( + defaultNamespace.getCatalogName()).listFunctions(likePattern, escape).stream() + .collect(toImmutableList())); + } + else { functions.addAll(SessionFunctionUtils.listFunctions(session.getSessionFunctions())); functions.addAll(functionNamespaceManagers.values().stream() .flatMap(manager -> manager.listFunctions(likePattern, escape).stream()) .collect(toImmutableList())); } - else { - functions.addAll(listBuiltInFunctions()); - } return functions.build().stream() .filter(function -> function.getVisibility() == PUBLIC || @@ -384,7 +407,7 @@ public Collection listBuiltInFunctions() public Collection getFunctions(Session session, QualifiedObjectName functionName) { - if (functionName.getCatalogSchemaName().equals(DEFAULT_NAMESPACE) && + if (functionName.getCatalogSchemaName().equals(JAVA_BUILTIN_NAMESPACE) && SessionFunctionUtils.listFunctionNames(session.getSessionFunctions()).contains(functionName.getObjectName())) { return SessionFunctionUtils.getFunctions(session.getSessionFunctions(), functionName); } @@ -442,7 +465,7 @@ public FunctionHandle resolveFunction( QualifiedObjectName functionName, List parameterTypes) { - if (functionName.getCatalogSchemaName().equals(DEFAULT_NAMESPACE)) { + if (functionName.getCatalogSchemaName().equals(JAVA_BUILTIN_NAMESPACE)) { if (sessionFunctions.isPresent()) { Collection candidates = SessionFunctionUtils.getFunctions(sessionFunctions.get(), functionName); Optional match = functionSignatureMatcher.match(candidates, parameterTypes, true); @@ -598,17 +621,28 @@ public boolean nullIfSpecialFormEnabled() public FunctionHandle lookupFunction(String name, List parameterTypes) { QualifiedObjectName functionName = getFunctionAndTypeResolver().qualifyObjectName(QualifiedName.of(name)); + return lookupFunction(functionName, parameterTypes); + } + + public FunctionHandle lookupFunction(QualifiedObjectName functionName, List parameterTypes) + { + Optional> functionNamespaceManager = getServingFunctionNamespaceManager(functionName.getCatalogSchemaName()); + if (!functionNamespaceManager.isPresent()) { + throw new PrestoException(FUNCTION_NOT_FOUND, format("Cannot find function namespace for function '%s'", functionName)); + } + checkArgument(functionName.getCatalogSchemaName().equals(defaultNamespace) || + functionName.getCatalogSchemaName().equals(JAVA_BUILTIN_NAMESPACE), "Only default/built-in function namespace managers are allowed."); if (parameterTypes.stream().noneMatch(TypeSignatureProvider::hasDependency)) { return lookupCachedFunction(functionName, parameterTypes); } - Collection candidates = builtInTypeAndFunctionNamespaceManager.getFunctions(Optional.empty(), functionName); + Collection candidates = functionNamespaceManager.get().getFunctions(Optional.empty(), functionName); Optional match = functionSignatureMatcher.match(candidates, parameterTypes, false); if (!match.isPresent()) { throw new PrestoException(FUNCTION_NOT_FOUND, constructFunctionNotFoundErrorMessage(functionName, parameterTypes, candidates)); } - return builtInTypeAndFunctionNamespaceManager.getFunctionHandle(Optional.empty(), match.get()); + return functionNamespaceManager.get().getFunctionHandle(Optional.empty(), match.get()); } public FunctionHandle lookupCast(CastType castType, Type fromType, Type toType) @@ -631,6 +665,11 @@ public FunctionHandle lookupCast(CastType castType, Type fromType, Type toType) return builtInTypeAndFunctionNamespaceManager.getFunctionHandle(Optional.empty(), signature); } + public CatalogSchemaName getDefaultNamespace() + { + return defaultNamespace; + } + protected Type getType(UserDefinedType userDefinedType) { // Distinct type @@ -698,7 +737,8 @@ private FunctionHandle resolveFunctionInternal(Optional transacti private FunctionHandle resolveBuiltInFunction(QualifiedObjectName functionName, List parameterTypes) { - checkArgument(functionName.getCatalogSchemaName().equals(DEFAULT_NAMESPACE), "Expect built-in functions"); + checkArgument(functionName.getCatalogSchemaName().equals(defaultNamespace) || + functionName.getCatalogSchemaName().equals(JAVA_BUILTIN_NAMESPACE), "Expect built-in/default namespace functions"); checkArgument(parameterTypes.stream().noneMatch(TypeSignatureProvider::hasDependency), "Expect parameter types not to have dependency"); return resolveFunctionInternal(Optional.empty(), functionName, parameterTypes); } @@ -726,6 +766,75 @@ private Optional> getServingFunc return Optional.ofNullable(functionNamespaceManagers.get(typeSignatureBase.getTypeName().getCatalogName())); } + @Override + @SuppressWarnings("unchecked") + public SpecializedFunctionKey getSpecializedFunctionKey(Signature signature) + { + QualifiedObjectName functionName = signature.getName(); + Optional> functionNamespaceManager = getServingFunctionNamespaceManager(functionName.getCatalogSchemaName()); + if (!functionNamespaceManager.isPresent()) { + throw new PrestoException(FUNCTION_NOT_FOUND, format("Cannot find function namespace for signature '%s'", functionName)); + } + + Collection candidates = (Collection) functionNamespaceManager.get().getFunctions(Optional.empty(), functionName); + + // search for exact match + Type returnType = getType(signature.getReturnType()); + List argumentTypeSignatureProviders = fromTypeSignatures(signature.getArgumentTypes()); + for (SqlFunction candidate : candidates) { + Optional boundVariables = new SignatureBinder(this, candidate.getSignature(), false) + .bindVariables(argumentTypeSignatureProviders, returnType); + if (boundVariables.isPresent()) { + return new SpecializedFunctionKey(candidate, boundVariables.get(), argumentTypeSignatureProviders.size()); + } + } + + // TODO: hack because there could be "type only" coercions (which aren't necessarily included as implicit casts), + // so do a second pass allowing "type only" coercions + List argumentTypes = resolveTypes(signature.getArgumentTypes(), this); + for (SqlFunction candidate : candidates) { + SignatureBinder binder = new SignatureBinder(this, candidate.getSignature(), true); + Optional boundVariables = binder.bindVariables(argumentTypeSignatureProviders, returnType); + if (!boundVariables.isPresent()) { + continue; + } + Signature boundSignature = applyBoundVariables(candidate.getSignature(), boundVariables.get(), argumentTypes.size()); + + if (!isTypeOnlyCoercion(getType(boundSignature.getReturnType()), returnType)) { + continue; + } + boolean nonTypeOnlyCoercion = false; + for (int i = 0; i < argumentTypes.size(); i++) { + Type expectedType = getType(boundSignature.getArgumentTypes().get(i)); + if (!isTypeOnlyCoercion(argumentTypes.get(i), expectedType)) { + nonTypeOnlyCoercion = true; + break; + } + } + if (nonTypeOnlyCoercion) { + continue; + } + + return new SpecializedFunctionKey(candidate, boundVariables.get(), argumentTypes.size()); + } + + // One final check for magic literal functions. + // Magic literal functions are only present in the JAVA_BUILTIN_NAMESPACE function namespace. + if (!signature.getName().getCatalogSchemaName().equals(JAVA_BUILTIN_NAMESPACE)) { + throw new PrestoException(FUNCTION_IMPLEMENTATION_MISSING, format("%s not found", signature)); + } + return builtInTypeAndFunctionNamespaceManager.doGetSpecializedFunctionKeyForMagicLiteralFunctions(signature, this); + } + + public CatalogSchemaName configureDefaultNamespace(String defaultNamespacePrefixString) + { + if (!defaultNamespacePrefixString.matches(DEFAULT_NAMESPACE_PREFIX_PATTERN.pattern())) { + throw new PrestoException(GENERIC_USER_ERROR, format("Default namespace prefix string should be in the form of 'catalog.schema', found: %s", defaultNamespacePrefixString)); + } + String[] catalogSchemaNameString = defaultNamespacePrefixString.split("\\."); + return new CatalogSchemaName(catalogSchemaNameString[0], catalogSchemaNameString[1]); + } + private static class FunctionResolutionCacheKey { private final QualifiedObjectName functionName; diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionSignatureMatcher.java b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionSignatureMatcher.java index 5021ef3d9f5e7..7c71047a3b107 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/FunctionSignatureMatcher.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/FunctionSignatureMatcher.java @@ -32,7 +32,7 @@ import java.util.stream.Collectors; import static com.facebook.presto.common.type.UnknownType.UNKNOWN; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.spi.StandardErrorCode.AMBIGUOUS_FUNCTION_CALL; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypeSignatures; @@ -308,7 +308,7 @@ static String constructFunctionNotFoundErrorMessage(QualifiedObjectName function private static String toConciseFunctionName(QualifiedObjectName functionName) { - if (functionName.getCatalogSchemaName().equals(DEFAULT_NAMESPACE)) { + if (functionName.getCatalogSchemaName().equals(JAVA_BUILTIN_NAMESPACE)) { return functionName.getObjectName(); } return functionName.toString(); diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/SessionFunctionHandle.java b/presto-main/src/main/java/com/facebook/presto/metadata/SessionFunctionHandle.java index 5ac4b9858336b..6e0cf4b69be55 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/SessionFunctionHandle.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/SessionFunctionHandle.java @@ -14,6 +14,7 @@ package com.facebook.presto.metadata; import com.facebook.presto.common.CatalogSchemaName; +import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.FunctionImplementationType; import com.facebook.presto.spi.function.FunctionKind; @@ -26,6 +27,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; import java.util.Objects; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -62,6 +64,12 @@ public FunctionKind getKind() return sqlFunction.getSignature().getKind(); } + @Override + public List getArgumentTypes() + { + return sqlFunction.getSignature().getArgumentTypes(); + } + public FunctionMetadata getFunctionMetadata() { Signature signature = sqlFunction.getSignature(); diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/SignatureBinder.java b/presto-main/src/main/java/com/facebook/presto/metadata/SignatureBinder.java index 31572056413a7..42fdb77c3bdb5 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/SignatureBinder.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/SignatureBinder.java @@ -304,7 +304,7 @@ private boolean appendConstraintSolvers( // * type with type parameter of type/named_type kind (except function type) if (FunctionType.NAME.equals(formalTypeSignature.getBase())) { - List formalTypeParameterTypeSignatures = formalTypeSignature.getTypeParametersAsTypeSignatures(); + List formalTypeParameterTypeSignatures = formalTypeSignature.getTypeOrNamedTypeParametersAsTypeSignatures(); resultBuilder.add(new FunctionSolver( getLambdaArgumentTypeSignatures(formalTypeSignature), formalTypeParameterTypeSignatures.get(formalTypeParameterTypeSignatures.size() - 1), @@ -551,7 +551,7 @@ else if (fromType instanceof TypeWithName) { private static List getLambdaArgumentTypeSignatures(TypeSignature lambdaTypeSignature) { - List typeParameters = lambdaTypeSignature.getTypeParametersAsTypeSignatures(); + List typeParameters = lambdaTypeSignature.getTypeOrNamedTypeParametersAsTypeSignatures(); return typeParameters.subList(0, typeParameters.size() - 1); } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/SignatureBuilder.java b/presto-main/src/main/java/com/facebook/presto/metadata/SignatureBuilder.java index c04b664f0d1c8..d58480a5fc15b 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/SignatureBuilder.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/SignatureBuilder.java @@ -24,7 +24,7 @@ import java.util.List; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; @@ -49,7 +49,7 @@ public static SignatureBuilder builder() public SignatureBuilder name(String name) { - this.name = QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, requireNonNull(name, "name is null")); + this.name = QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, requireNonNull(name, "name is null")); return this; } diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/SpecializedFunctionKey.java b/presto-main/src/main/java/com/facebook/presto/metadata/SpecializedFunctionKey.java index 62e052af08af4..f81018a760670 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/SpecializedFunctionKey.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/SpecializedFunctionKey.java @@ -15,6 +15,7 @@ import com.facebook.presto.spi.function.Signature; import com.facebook.presto.spi.function.SqlFunction; +import com.facebook.presto.spi.function.SqlFunctionSupplier; import java.util.Objects; @@ -22,6 +23,7 @@ import static java.util.Objects.requireNonNull; public class SpecializedFunctionKey + implements SqlFunctionSupplier { private final SqlFunction function; private final BoundVariables boundVariables; diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/SqlAggregationFunction.java b/presto-main/src/main/java/com/facebook/presto/metadata/SqlAggregationFunction.java index 07a5789009370..9d2b6c0273632 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/SqlAggregationFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/SqlAggregationFunction.java @@ -26,7 +26,7 @@ import java.util.List; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.spi.function.FunctionKind.AGGREGATE; import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static com.google.common.base.Preconditions.checkArgument; @@ -94,7 +94,7 @@ private static Signature createSignature( requireNonNull(argumentTypes, "argumentTypes is null"); checkArgument(kind == AGGREGATE, "kind must be an aggregate"); return new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, name), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, name), kind, ImmutableList.copyOf(typeVariableConstraints), ImmutableList.copyOf(longVariableConstraints), diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceStore.java b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceStore.java index d74d74c4c805d..85e2d1118734a 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceStore.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/StaticFunctionNamespaceStore.java @@ -14,6 +14,7 @@ package com.facebook.presto.metadata; import com.facebook.airlift.log.Logger; +import com.facebook.presto.spi.NodeManager; import com.google.common.collect.ImmutableList; import javax.inject.Inject; @@ -33,15 +34,16 @@ public class StaticFunctionNamespaceStore { private static final Logger log = Logger.get(StaticFunctionNamespaceStore.class); private static final String FUNCTION_NAMESPACE_MANAGER_NAME = "function-namespace-manager.name"; - private final FunctionAndTypeManager functionAndTypeManager; + private final NodeManager nodeManager; private final File configDir; private final AtomicBoolean functionNamespaceLoading = new AtomicBoolean(); @Inject - public StaticFunctionNamespaceStore(FunctionAndTypeManager functionAndTypeManager, StaticFunctionNamespaceStoreConfig config) + public StaticFunctionNamespaceStore(FunctionAndTypeManager functionAndTypeManager, NodeManager nodeManager, StaticFunctionNamespaceStoreConfig config) { this.functionAndTypeManager = functionAndTypeManager; + this.nodeManager = nodeManager; this.configDir = config.getFunctionNamespaceConfigurationDir(); } @@ -77,8 +79,7 @@ private void loadFunctionNamespaceManager(String catalogName, Map(properties); String functionNamespaceManagerName = properties.remove(FUNCTION_NAMESPACE_MANAGER_NAME); checkState(!isNullOrEmpty(functionNamespaceManagerName), "%s property must be present", FUNCTION_NAMESPACE_MANAGER_NAME); - - functionAndTypeManager.loadFunctionNamespaceManager(functionNamespaceManagerName, catalogName, properties); + functionAndTypeManager.loadFunctionNamespaceManager(functionNamespaceManagerName, catalogName, properties, nodeManager); log.info("-- Added function namespace manager [%s] --", catalogName); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AggregationImplementation.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AggregationImplementation.java index ad7c17b08b543..8cb42304cd67f 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AggregationImplementation.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/AggregationImplementation.java @@ -46,7 +46,7 @@ import java.util.stream.Stream; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.annotations.FunctionsParserHelper.containsAnnotation; import static com.facebook.presto.operator.annotations.FunctionsParserHelper.createTypeVariableConstraints; import static com.facebook.presto.operator.annotations.FunctionsParserHelper.parseLiteralParameters; @@ -306,7 +306,7 @@ private Parser( private AggregationImplementation get() { Signature signature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, header.getName()), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, header.getName()), FunctionKind.AGGREGATE, typeVariableConstraints, longVariableConstraints, diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeStatisticalDigestFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeStatisticalDigestFunction.java index c4921d23d1a80..a47776c97c954 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeStatisticalDigestFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/MergeStatisticalDigestFunction.java @@ -35,7 +35,7 @@ import java.util.List; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.aggregation.AggregationUtils.generateAggregationName; import static com.facebook.presto.spi.function.FunctionKind.AGGREGATE; import static com.facebook.presto.spi.function.Signature.comparableTypeParameter; @@ -68,7 +68,7 @@ public abstract class MergeStatisticalDigestFunction MergeStatisticalDigestFunction(String name, String type, StatisticalDigestStateFactory factory, SqlFunctionVisibility visibility) { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, name), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, name), AGGREGATE, ImmutableList.of(comparableTypeParameter("T")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/StatisticalDigestAggregationFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/StatisticalDigestAggregationFunction.java index 7e62e2a6a4111..ba2cfe3b4cacb 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/aggregation/StatisticalDigestAggregationFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/aggregation/StatisticalDigestAggregationFunction.java @@ -43,7 +43,7 @@ import static com.facebook.presto.common.type.StandardTypes.QDIGEST; import static com.facebook.presto.common.type.StandardTypes.TDIGEST; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.aggregation.AggregationUtils.generateAggregationName; import static com.facebook.presto.operator.scalar.QuantileDigestFunctions.DEFAULT_ACCURACY; import static com.facebook.presto.operator.scalar.TDigestFunctions.DEFAULT_COMPRESSION; @@ -110,7 +110,7 @@ public abstract class StatisticalDigestAggregationFunction StatisticalDigestAggregationFunction(String name, String type, StatisticalDigestStateFactory factory, SqlFunctionVisibility visibility, TypeSignature... typeSignatures) { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, name), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, name), AGGREGATE, ImmutableList.of(comparableTypeParameter("V")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ApplyFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ApplyFunction.java index cd42b25ce9b9a..6160da8b68f09 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ApplyFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ApplyFunction.java @@ -27,7 +27,7 @@ import java.lang.invoke.MethodHandle; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.functionTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.USE_BOXED_TYPE; @@ -49,7 +49,7 @@ public final class ApplyFunction private ApplyFunction() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "apply"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "apply"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T"), typeVariable("U")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConcatFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConcatFunction.java index f64ecefa10993..dc6f0da3bde62 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConcatFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConcatFunction.java @@ -33,7 +33,7 @@ import java.util.Optional; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; @@ -58,7 +58,7 @@ public final class ArrayConcatFunction private ArrayConcatFunction() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, FUNCTION_NAME), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, FUNCTION_NAME), FunctionKind.SCALAR, ImmutableList.of(typeVariable("E")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConstructor.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConstructor.java index d36fb47f77115..cefbbf2e3c0a6 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConstructor.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayConstructor.java @@ -54,7 +54,7 @@ import static com.facebook.presto.bytecode.expression.BytecodeExpressions.constantNull; import static com.facebook.presto.bytecode.expression.BytecodeExpressions.equal; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.USE_BOXED_TYPE; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; @@ -76,7 +76,7 @@ public final class ArrayConstructor public ArrayConstructor() { - super(new Signature(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "array_constructor"), + super(new Signature(QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "array_constructor"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("E")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayFlattenFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayFlattenFunction.java index 3a3f537dc209a..47520216b0599 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayFlattenFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayFlattenFunction.java @@ -33,7 +33,7 @@ import java.util.Optional; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.Signature.typeVariable; @@ -51,7 +51,7 @@ public class ArrayFlattenFunction private ArrayFlattenFunction() { - super(new Signature(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, FUNCTION_NAME), + super(new Signature(QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, FUNCTION_NAME), FunctionKind.SCALAR, ImmutableList.of(typeVariable("E")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayJoin.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayJoin.java index 8811c2c1ecfcc..6bec808510cce 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayJoin.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayJoin.java @@ -41,7 +41,7 @@ import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.common.type.VarcharType.VARCHAR; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.metadata.CastType.CAST; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; @@ -112,7 +112,7 @@ public static class ArrayJoinWithNullReplacement public ArrayJoinWithNullReplacement() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, FUNCTION_NAME), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, FUNCTION_NAME), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T")), ImmutableList.of(), @@ -154,7 +154,7 @@ public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariab public ArrayJoin() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, FUNCTION_NAME), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, FUNCTION_NAME), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayReduceFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayReduceFunction.java index 8f1b534675434..adb6be974c556 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayReduceFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayReduceFunction.java @@ -31,7 +31,7 @@ import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.common.type.TypeUtils.readNativeValue; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.functionTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; @@ -50,7 +50,7 @@ public final class ArrayReduceFunction private ArrayReduceFunction() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "reduce"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "reduce"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T"), typeVariable("S"), typeVariable("R")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToElementConcatFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToElementConcatFunction.java index 7d2002ee4e0b4..eb46a4f2ffbcd 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToElementConcatFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayToElementConcatFunction.java @@ -28,7 +28,7 @@ import java.lang.invoke.MethodHandle; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.Signature.typeVariable; @@ -49,7 +49,7 @@ public class ArrayToElementConcatFunction public ArrayToElementConcatFunction() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "concat"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "concat"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("E")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayTransformFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayTransformFunction.java index b524a822d1137..bba0342054582 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayTransformFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ArrayTransformFunction.java @@ -62,7 +62,7 @@ import static com.facebook.presto.bytecode.instruction.VariableInstruction.incrementVariable; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.common.type.UnknownType.UNKNOWN; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.functionTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; @@ -82,7 +82,7 @@ public final class ArrayTransformFunction private ArrayTransformFunction() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "transform"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "transform"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T"), typeVariable("U")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ConcatFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ConcatFunction.java index fdfe16843b57f..a1c20f92961c4 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ConcatFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ConcatFunction.java @@ -51,7 +51,7 @@ import static com.facebook.presto.bytecode.expression.BytecodeExpressions.invokeStatic; import static com.facebook.presto.common.type.VarbinaryType.VARBINARY; import static com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; @@ -76,7 +76,7 @@ public final class ConcatFunction private ConcatFunction(TypeSignature type, String description) { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "concat"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "concat"), FunctionKind.SCALAR, ImmutableList.of(), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ElementToArrayConcatFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ElementToArrayConcatFunction.java index 83c6909800526..7cf18212a5b7b 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ElementToArrayConcatFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ElementToArrayConcatFunction.java @@ -28,7 +28,7 @@ import java.lang.invoke.MethodHandle; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.Signature.typeVariable; @@ -49,7 +49,7 @@ public class ElementToArrayConcatFunction public ElementToArrayConcatFunction() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "concat"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "concat"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("E")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/Greatest.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/Greatest.java index 384cf57ff3132..cc27f1e73f623 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/Greatest.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/Greatest.java @@ -16,7 +16,7 @@ import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.function.OperatorType; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; public final class Greatest extends AbstractGreatestLeast @@ -25,7 +25,7 @@ public final class Greatest public Greatest() { - super(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "greatest"), OperatorType.GREATER_THAN); + super(QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "greatest"), OperatorType.GREATER_THAN); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/InvokeFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/InvokeFunction.java index cfdd76c4b122c..d9a27a2d5cef8 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/InvokeFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/InvokeFunction.java @@ -28,7 +28,7 @@ import java.lang.invoke.MethodHandle; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.functionTypeArgumentProperty; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; @@ -48,7 +48,7 @@ public final class InvokeFunction private InvokeFunction() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "invoke"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "invoke"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToArrayCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToArrayCast.java index c365d0d8975c9..e96c35d994f12 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToArrayCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToArrayCast.java @@ -23,7 +23,7 @@ import com.google.common.collect.ImmutableList; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.JsonToArrayCast.JSON_TO_ARRAY; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.spi.function.Signature.typeVariable; @@ -38,7 +38,7 @@ public final class JsonStringToArrayCast private JsonStringToArrayCast() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, JSON_STRING_TO_ARRAY_NAME), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, JSON_STRING_TO_ARRAY_NAME), SCALAR, ImmutableList.of(typeVariable("T")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToMapCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToMapCast.java index 3c3070a70b763..404aeb8791929 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToMapCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToMapCast.java @@ -23,7 +23,7 @@ import com.google.common.collect.ImmutableList; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.JsonToMapCast.JSON_TO_MAP; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.spi.function.Signature.comparableTypeParameter; @@ -39,7 +39,7 @@ public final class JsonStringToMapCast private JsonStringToMapCast() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, JSON_STRING_TO_MAP_NAME), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, JSON_STRING_TO_MAP_NAME), SCALAR, ImmutableList.of(comparableTypeParameter("K"), typeVariable("V")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToRowCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToRowCast.java index 3e1a59b5e6dc0..5d816bd022ce4 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToRowCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonStringToRowCast.java @@ -23,7 +23,7 @@ import com.google.common.collect.ImmutableList; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.JsonToRowCast.JSON_TO_ROW; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.spi.function.Signature.withVariadicBound; @@ -38,7 +38,7 @@ public final class JsonStringToRowCast private JsonStringToRowCast() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, JSON_STRING_TO_ROW_NAME), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, JSON_STRING_TO_ROW_NAME), SCALAR, ImmutableList.of(withVariadicBound("T", "row")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/KDistinct.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/KDistinct.java index 5ba6298bff124..7fb42536aa21f 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/KDistinct.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/KDistinct.java @@ -30,7 +30,7 @@ import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.functionTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; @@ -46,7 +46,7 @@ public final class KDistinct private KDistinct() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "k_distinct"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "k_distinct"), FunctionKind.SCALAR, ImmutableList.of(), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/Least.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/Least.java index da579ede93320..ab1a735fa9349 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/Least.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/Least.java @@ -16,7 +16,7 @@ import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.function.OperatorType; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; public final class Least extends AbstractGreatestLeast @@ -25,7 +25,7 @@ public final class Least public Least() { - super(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "least"), OperatorType.LESS_THAN); + super(QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "least"), OperatorType.LESS_THAN); } @Override diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConcatFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConcatFunction.java index 1e56f3bd5fa63..989a990c43cd0 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConcatFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConcatFunction.java @@ -40,7 +40,7 @@ import java.util.Optional; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; @@ -66,7 +66,7 @@ public final class MapConcatFunction private MapConcatFunction() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, FUNCTION_NAME), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, FUNCTION_NAME), FunctionKind.SCALAR, ImmutableList.of(typeVariable("K"), typeVariable("V")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConstructor.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConstructor.java index 0dee32f9b7102..3a22c56f5b98c 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConstructor.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapConstructor.java @@ -45,7 +45,7 @@ import static com.facebook.presto.common.function.OperatorType.INDETERMINATE; import static com.facebook.presto.common.type.StandardTypes.MAP; import static com.facebook.presto.common.type.TypeUtils.readNativeValue; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; @@ -84,7 +84,7 @@ public final class MapConstructor public MapConstructor() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "map"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "map"), FunctionKind.SCALAR, ImmutableList.of(comparableTypeParameter("K"), typeVariable("V")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapElementAtFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapElementAtFunction.java index 57dd1949a7055..276c355f31856 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapElementAtFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapElementAtFunction.java @@ -37,7 +37,7 @@ import static com.facebook.presto.common.block.MethodHandleUtil.nativeValueGetter; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.common.type.TypeUtils.readNativeValue; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; @@ -60,7 +60,7 @@ public class MapElementAtFunction protected MapElementAtFunction() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "element_at"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "element_at"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("K"), typeVariable("V")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformKeyFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformKeyFunction.java index 9ce86310bf6be..4809bc6b0f118 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformKeyFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformKeyFunction.java @@ -72,7 +72,7 @@ import static com.facebook.presto.bytecode.instruction.VariableInstruction.incrementVariable; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.common.type.UnknownType.UNKNOWN; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.functionTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; @@ -91,7 +91,7 @@ public final class MapTransformKeyFunction private MapTransformKeyFunction() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "transform_keys"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "transform_keys"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("K1"), typeVariable("K2"), typeVariable("V")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformValueFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformValueFunction.java index a884c9d360700..b4daed54c4eac 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformValueFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapTransformValueFunction.java @@ -75,7 +75,7 @@ import static com.facebook.presto.bytecode.instruction.VariableInstruction.incrementVariable; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.common.type.UnknownType.UNKNOWN; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.functionTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; @@ -96,7 +96,7 @@ public final class MapTransformValueFunction private MapTransformValueFunction() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "transform_values"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "transform_values"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("K"), typeVariable("V1"), typeVariable("V2")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapZipWithFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapZipWithFunction.java index 100c161d28f16..b7f74e3362c13 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapZipWithFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/MapZipWithFunction.java @@ -46,7 +46,7 @@ import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.common.type.TypeUtils.readNativeValue; import static com.facebook.presto.common.type.TypeUtils.writeNativeValue; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.functionTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; @@ -69,7 +69,7 @@ public final class MapZipWithFunction private MapZipWithFunction() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "map_zip_with"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "map_zip_with"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("K"), typeVariable("V1"), typeVariable("V2"), typeVariable("V3")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/TryCastFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/TryCastFunction.java index a1386344426fe..061c6dfca9fe1 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/TryCastFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/TryCastFunction.java @@ -30,7 +30,7 @@ import java.util.List; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.metadata.CastType.CAST; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; @@ -49,7 +49,7 @@ public class TryCastFunction public TryCastFunction() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, TRY_CAST_NAME), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, TRY_CAST_NAME), FunctionKind.SCALAR, ImmutableList.of(typeVariable("F"), typeVariable("T")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipFunction.java index 539bba526aec7..097bd9f22dfc9 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipFunction.java @@ -34,7 +34,7 @@ import java.util.stream.IntStream; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; @@ -70,7 +70,7 @@ private ZipFunction(int arity) private ZipFunction(List typeParameters) { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "zip"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "zip"), FunctionKind.SCALAR, typeParameters.stream().map(Signature::typeVariable).collect(toImmutableList()), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipWithFunction.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipWithFunction.java index 1ce21b169d21e..0c0243dd08227 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipWithFunction.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/ZipWithFunction.java @@ -38,7 +38,7 @@ import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.common.type.TypeUtils.readNativeValue; import static com.facebook.presto.common.type.TypeUtils.writeNativeValue; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.functionTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; @@ -59,7 +59,7 @@ public final class ZipWithFunction private ZipWithFunction() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "zip_with"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "zip_with"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T"), typeVariable("U"), typeVariable("R")), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/CodegenScalarFromAnnotationsParser.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/CodegenScalarFromAnnotationsParser.java index f773497fd08bb..ae83d270b8308 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/CodegenScalarFromAnnotationsParser.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/CodegenScalarFromAnnotationsParser.java @@ -49,7 +49,7 @@ import java.util.Set; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.metadata.SignatureBinder.applyBoundVariables; import static com.facebook.presto.operator.annotations.FunctionsParserHelper.findPublicStaticMethods; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.functionTypeArgumentProperty; @@ -114,7 +114,7 @@ private static SqlScalarFunction createSqlScalarFunction(Method method) CodegenScalarFunction codegenScalarFunction = method.getAnnotation(CodegenScalarFunction.class); Signature signature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, codegenScalarFunction.value()), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, codegenScalarFunction.value()), FunctionKind.SCALAR, Arrays.stream(method.getAnnotationsByType(TypeParameter.class)).map(t -> withVariadicBound(t.value(), t.boundedBy().isEmpty() ? null : t.boundedBy())).collect(toImmutableList()), ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/OperatorValidator.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/OperatorValidator.java index a0249ce116030..edfc6b9260c0c 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/OperatorValidator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/OperatorValidator.java @@ -59,11 +59,11 @@ public static void validateOperator(OperatorType operatorType, TypeSignature ret checkArgument(argumentTypes.get(0).getBase().equals(StandardTypes.ARRAY) || argumentTypes.get(0).getBase().equals(StandardTypes.MAP), "First argument must be an ARRAY or MAP"); if (argumentTypes.get(0).getBase().equals(StandardTypes.ARRAY)) { checkArgument(argumentTypes.get(1).getBase().equals(StandardTypes.BIGINT), "Second argument must be a BIGINT"); - TypeSignature elementType = argumentTypes.get(0).getTypeParametersAsTypeSignatures().get(0); + TypeSignature elementType = argumentTypes.get(0).getTypeOrNamedTypeParametersAsTypeSignatures().get(0); checkArgument(returnType.equals(elementType), "[] return type does not match ARRAY element type"); } else { - TypeSignature valueType = argumentTypes.get(0).getTypeParametersAsTypeSignatures().get(1); + TypeSignature valueType = argumentTypes.get(0).getTypeOrNamedTypeParametersAsTypeSignatures().get(1); checkArgument(returnType.equals(valueType), "[] return type does not match MAP value type"); } break; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarImplementationHeader.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarImplementationHeader.java index 0a83dc4233d49..b79b0b7f05219 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarImplementationHeader.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/ScalarImplementationHeader.java @@ -26,7 +26,7 @@ import java.util.List; import java.util.Optional; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.annotations.FunctionsParserHelper.parseDescription; import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN; import static com.google.common.base.CaseFormat.LOWER_CAMEL; @@ -42,7 +42,7 @@ public class ScalarImplementationHeader private ScalarImplementationHeader(String name, ScalarHeader header) { - this.name = QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, requireNonNull(name)); + this.name = QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, requireNonNull(name)); this.operatorType = Optional.empty(); this.header = requireNonNull(header); } diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/SqlInvokedScalarFromAnnotationsParser.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/SqlInvokedScalarFromAnnotationsParser.java index 2517bc19b9080..ee866caecfbda 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/SqlInvokedScalarFromAnnotationsParser.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/annotations/SqlInvokedScalarFromAnnotationsParser.java @@ -39,7 +39,7 @@ import java.util.stream.Stream; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.annotations.FunctionsParserHelper.findPublicStaticMethods; import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_IMPLEMENTATION_ERROR; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; @@ -166,13 +166,14 @@ else if (method.isAnnotationPresent(SqlParameters.class)) { return Stream.concat(Stream.of(functionHeader.value()), stream(functionHeader.alias())) .map(name -> new SqlInvokedFunction( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, name), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, name), parameters, typeVariableConstraints, returnType, functionDescription, routineCharacteristics, body, + false, notVersioned(), SCALAR, Optional.empty())) diff --git a/presto-main/src/main/java/com/facebook/presto/operator/window/ReflectionWindowFunctionSupplier.java b/presto-main/src/main/java/com/facebook/presto/operator/window/ReflectionWindowFunctionSupplier.java index d4a63bdcb86a2..07dd7cc5e3f3c 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/window/ReflectionWindowFunctionSupplier.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/window/ReflectionWindowFunctionSupplier.java @@ -25,7 +25,7 @@ import java.lang.reflect.Constructor; import java.util.List; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.spi.function.FunctionKind.WINDOW; import static java.util.Objects.requireNonNull; @@ -36,7 +36,7 @@ public class ReflectionWindowFunctionSupplier public ReflectionWindowFunctionSupplier(String name, Type returnType, List argumentTypes, Class type) { - this(new Signature(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, name), WINDOW, returnType.getTypeSignature(), Lists.transform(argumentTypes, Type::getTypeSignature)), type); + this(new Signature(QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, name), WINDOW, returnType.getTypeSignature(), Lists.transform(argumentTypes, Type::getTypeSignature)), type); } public ReflectionWindowFunctionSupplier(Signature signature, Class type) diff --git a/presto-main/src/main/java/com/facebook/presto/operator/window/WindowAnnotationsParser.java b/presto-main/src/main/java/com/facebook/presto/operator/window/WindowAnnotationsParser.java index 43c4c51eb9ea9..fff2f8865578e 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/window/WindowAnnotationsParser.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/window/WindowAnnotationsParser.java @@ -25,7 +25,7 @@ import java.util.stream.Stream; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.spi.function.FunctionKind.WINDOW; import static com.facebook.presto.spi.function.Signature.typeVariable; import static com.google.common.base.Preconditions.checkArgument; @@ -58,7 +58,7 @@ private static SqlWindowFunction parse( .collect(toImmutableList()); Signature signature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, window.name()), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, window.name()), WINDOW, typeVariables, ImmutableList.of(), diff --git a/presto-main/src/main/java/com/facebook/presto/server/protocol/QueryResourceUtil.java b/presto-main/src/main/java/com/facebook/presto/server/protocol/QueryResourceUtil.java index b713d7b60cac7..702e33108480e 100644 --- a/presto-main/src/main/java/com/facebook/presto/server/protocol/QueryResourceUtil.java +++ b/presto-main/src/main/java/com/facebook/presto/server/protocol/QueryResourceUtil.java @@ -352,13 +352,13 @@ public static Object parseToJson(TypeSignature signature, Object value) if (signature.getBase().equals(ARRAY)) { List parsedValue = new ArrayList<>(); for (Object object : List.class.cast(value)) { - parsedValue.add(parseToJson(signature.getTypeParametersAsTypeSignatures().get(0), object)); + parsedValue.add(parseToJson(signature.getTypeOrNamedTypeParametersAsTypeSignatures().get(0), object)); } return LIST_JSON_CODEC.toJson(parsedValue); } if (signature.getBase().equals(MAP)) { - TypeSignature keySignature = signature.getTypeParametersAsTypeSignatures().get(0); - TypeSignature valueSignature = signature.getTypeParametersAsTypeSignatures().get(1); + TypeSignature keySignature = signature.getTypeOrNamedTypeParametersAsTypeSignatures().get(0); + TypeSignature valueSignature = signature.getTypeOrNamedTypeParametersAsTypeSignatures().get(1); Map parsedValue = new HashMap<>(Map.class.cast(value).size()); for (Map.Entry entry : (Set>) Map.class.cast(value).entrySet()) { parsedValue.put(parseToJson(keySignature, entry.getKey()), parseToJson(valueSignature, entry.getValue())); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java index f892bdf4d39f7..3dcc4bce7597a 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/ExpressionAnalyzer.java @@ -151,7 +151,7 @@ import static com.facebook.presto.common.type.UnknownType.UNKNOWN; import static com.facebook.presto.common.type.VarbinaryType.VARBINARY; import static com.facebook.presto.common.type.VarcharType.VARCHAR; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.spi.StandardErrorCode.OPERATOR_NOT_FOUND; import static com.facebook.presto.spi.StandardWarningCode.SEMANTIC_WARNING; import static com.facebook.presto.sql.NodeUtils.getSortItemsFromOrderBy; @@ -1145,7 +1145,7 @@ else if (frame.getType() == GROUPS) { } resolvedFunctions.put(NodeRef.of(node), function); - if (functionMetadata.getName().equals(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "REDUCE_AGG"))) { + if (functionMetadata.getName().equals(QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "REDUCE_AGG"))) { Expression initialValueArg = node.getArguments().get(1); // For builtin reduce_agg, we make sure the initial value is not null as we cannot handle null properly now. diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FunctionsConfig.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FunctionsConfig.java index 917f2f4a330a8..542d09ba00470 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FunctionsConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FunctionsConfig.java @@ -21,6 +21,7 @@ import javax.validation.constraints.Min; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.sql.analyzer.RegexLibrary.JONI; public class FunctionsConfig @@ -46,6 +47,7 @@ public class FunctionsConfig private boolean warnOnPossibleNans; private boolean legacyCharToVarcharCoercion; private boolean legacyJsonCast = true; + private String defaultNamespacePrefix = JAVA_BUILTIN_NAMESPACE.toString(); @Config("deprecated.legacy-array-agg") public FunctionsConfig setLegacyArrayAgg(boolean legacyArrayAgg) @@ -53,6 +55,7 @@ public FunctionsConfig setLegacyArrayAgg(boolean legacyArrayAgg) this.legacyArrayAgg = legacyArrayAgg; return this; } + public boolean isLegacyArrayAgg() { return legacyArrayAgg; @@ -304,4 +307,17 @@ public boolean isLegacyJsonCast() { return legacyJsonCast; } + + @Config("presto.default-namespace") + @ConfigDescription("Specifies the default function namespace prefix") + public FunctionsConfig setDefaultNamespacePrefix(String defaultNamespacePrefix) + { + this.defaultNamespacePrefix = defaultNamespacePrefix; + return this; + } + + public String getDefaultNamespacePrefix() + { + return defaultNamespacePrefix; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LiteralEncoder.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LiteralEncoder.java index 7e90497131ffa..02bb5953e54e9 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LiteralEncoder.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LiteralEncoder.java @@ -71,7 +71,7 @@ import static com.facebook.presto.common.type.TinyintType.TINYINT; import static com.facebook.presto.common.type.UnknownType.UNKNOWN; import static com.facebook.presto.common.type.VarbinaryType.VARBINARY; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.sql.relational.Expressions.constantNull; @@ -314,7 +314,7 @@ public static Signature getMagicLiteralFunctionSignature(Type type) { TypeSignature argumentType = typeForMagicLiteral(type).getTypeSignature(); - return new Signature(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, MAGIC_LITERAL_FUNCTION_PREFIX + type.getTypeSignature()), + return new Signature(QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, MAGIC_LITERAL_FUNCTION_PREFIX + type.getTypeSignature()), SCALAR, type.getTypeSignature(), argumentType); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java index b2254ec86c314..b6034d0c95b93 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java @@ -48,6 +48,7 @@ import com.facebook.presto.index.IndexManager; import com.facebook.presto.memory.MemoryManagerConfig; import com.facebook.presto.metadata.AnalyzeTableHandle; +import com.facebook.presto.metadata.BuiltInFunctionHandle; import com.facebook.presto.metadata.ConnectorMetadataUpdaterManager; import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.metadata.Metadata; @@ -137,6 +138,7 @@ import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.JavaAggregationFunctionImplementation; +import com.facebook.presto.spi.function.SqlFunctionHandle; import com.facebook.presto.spi.function.SqlFunctionId; import com.facebook.presto.spi.function.SqlInvokedFunction; import com.facebook.presto.spi.function.aggregation.LambdaProvider; @@ -282,6 +284,7 @@ import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT; import static com.facebook.presto.expressions.RowExpressionNodeInliner.replaceExpression; import static com.facebook.presto.geospatial.SphericalGeographyUtils.sphericalDistance; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.DistinctLimitOperator.DistinctLimitOperatorFactory; import static com.facebook.presto.operator.NestedLoopBuildOperator.NestedLoopBuildOperatorFactory; import static com.facebook.presto.operator.NestedLoopJoinOperator.NestedLoopJoinOperatorFactory; @@ -322,6 +325,7 @@ import static com.facebook.presto.spi.plan.ProjectNode.Locality.REMOTE; import static com.facebook.presto.spi.relation.ExpressionOptimizer.Level.OPTIMIZED; import static com.facebook.presto.sql.analyzer.ExpressionTreeUtils.createSymbolReference; +import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; import static com.facebook.presto.sql.gen.LambdaBytecodeGenerator.compileLambdaProvider; import static com.facebook.presto.sql.planner.RowExpressionInterpreter.rowExpressionInterpreter; import static com.facebook.presto.sql.planner.SortExpressionExtractor.getSortExpressionContext; @@ -2856,12 +2860,21 @@ public PhysicalOperation visitTableFinish(TableFinishNode node, LocalExecutionPl ImmutableMap.Builder outputMapping = ImmutableMap.builder(); + // Todo: Fix this + // When switching the default function namespace manager, column statistics functions still need to use the presto.default functions, + // because the column statistics functions build an aggregation factory that depends on an instance of BuiltInAggregationImplementation. OperatorFactory statisticsAggregation = node.getStatisticsAggregation().map(aggregation -> { List groupingVariables = aggregation.getGroupingVariables(); + Map aggregationMap = + aggregation.getAggregations().entrySet() + .stream().collect( + ImmutableMap.toImmutableMap( + Map.Entry::getKey, + entry -> createAggregation(entry.getValue()))); if (groupingVariables.isEmpty()) { return createAggregationOperatorFactory( node.getId(), - aggregation.getAggregations(), + aggregationMap, FINAL, 0, outputMapping, @@ -2871,7 +2884,7 @@ public PhysicalOperation visitTableFinish(TableFinishNode node, LocalExecutionPl } return createHashAggregationOperatorFactory( node.getId(), - aggregation.getAggregations(), + aggregationMap, ImmutableSet.of(), groupingVariables, ImmutableList.of(), @@ -2954,6 +2967,45 @@ public PhysicalOperation visitUpdate(UpdateNode node, LocalExecutionPlanContext return new PhysicalOperation(operatorFactory, layout, context, source); } + private Aggregation createAggregation(Aggregation aggregation) + { + CallExpression callExpression = aggregation.getCall(); + List argumentTypes; + FunctionHandle functionHandle = callExpression.getFunctionHandle(); + checkState(functionHandle instanceof BuiltInFunctionHandle || functionHandle instanceof SqlFunctionHandle); + if (functionHandle instanceof BuiltInFunctionHandle) { + return aggregation; + } + else { + argumentTypes = ((SqlFunctionHandle) functionHandle).getFunctionId().getArgumentTypes(); + } + + return new Aggregation( + createCallExpression(callExpression, argumentTypes), + aggregation.getFilter(), + aggregation.getOrderBy(), + aggregation.isDistinct(), + aggregation.getMask()); + } + + private CallExpression createCallExpression(CallExpression callExpression, List argumentTypes) + { + FunctionAndTypeManager functionAndTypeManager = metadata.getFunctionAndTypeManager(); + List types = + argumentTypes + .stream() + .map(functionAndTypeManager::getType) + .collect(ImmutableList.toImmutableList()); + return new CallExpression( + callExpression.getSourceLocation(), + callExpression.getDisplayName(), + functionAndTypeManager.lookupFunction( + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, callExpression.getDisplayName()), + fromTypes(types)), + callExpression.getType(), + callExpression.getArguments()); + } + private List createColumnValueAndRowIdChannels(List variableReferenceExpressions, List columnValueAndRowIdSymbols) { Integer[] columnValueAndRowIdChannels = new Integer[columnValueAndRowIdSymbols.size()]; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/CombineApproxPercentileFunctions.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/CombineApproxPercentileFunctions.java index c60328c004798..1ed5ea0c57398 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/CombineApproxPercentileFunctions.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/CombineApproxPercentileFunctions.java @@ -20,7 +20,6 @@ import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; -import com.facebook.presto.metadata.BuiltInFunctionHandle; import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.plan.AggregationNode; @@ -111,8 +110,7 @@ private static int getPercentilePosition(FunctionHandle functionHandle) // approx_percentile(x, percentage, accuracy) -> arguments.size() == 3 && arguments.get(1) is Double // approx_percentile(x, w, percentage) -> arguments.size() == 3 && arguments.get(1) is BigInt // approx_percentile(x, w, percentage, accuracy) -> arguments.size() == 4 - checkState(functionHandle instanceof BuiltInFunctionHandle); - List argumentTypes = ((BuiltInFunctionHandle) functionHandle).getSignature().getArgumentTypes(); + List argumentTypes = functionHandle.getArgumentTypes(); if (argumentTypes.size() == 2 || (argumentTypes.size() == 3 && argumentTypes.get(1).getBase().equals(StandardTypes.DOUBLE))) { return 1; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RewriteSpatialPartitioningAggregation.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RewriteSpatialPartitioningAggregation.java index 1f0f47e3e41dd..0cd34a89cfc04 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RewriteSpatialPartitioningAggregation.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RewriteSpatialPartitioningAggregation.java @@ -37,7 +37,7 @@ import static com.facebook.presto.SystemSessionProperties.getHashPartitionCount; import static com.facebook.presto.common.type.IntegerType.INTEGER; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; import static com.facebook.presto.sql.planner.plan.AssignmentUtils.identityAssignments; import static com.facebook.presto.sql.planner.plan.Patterns.aggregation; @@ -64,7 +64,7 @@ public class RewriteSpatialPartitioningAggregation implements Rule { private static final TypeSignature GEOMETRY_TYPE_SIGNATURE = parseTypeSignature("Geometry"); - private static final QualifiedObjectName NAME = QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "spatial_partitioning"); + private static final QualifiedObjectName NAME = QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "spatial_partitioning"); private final Pattern pattern = aggregation().matching(this::hasSpatialPartitioningAggregation); private final Metadata metadata; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/KeyBasedSampler.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/KeyBasedSampler.java index b6f3f14bd6ae6..f34359aeba6af 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/KeyBasedSampler.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/KeyBasedSampler.java @@ -14,6 +14,7 @@ package com.facebook.presto.sql.planner.optimizations; import com.facebook.presto.Session; +import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.function.OperatorType; import com.facebook.presto.common.type.Type; import com.facebook.presto.common.type.Varchars; @@ -54,6 +55,7 @@ import static com.facebook.presto.common.type.BooleanType.BOOLEAN; import static com.facebook.presto.common.type.DoubleType.DOUBLE; import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.metadata.CastType.CAST; import static com.facebook.presto.spi.StandardErrorCode.FUNCTION_NOT_FOUND; import static com.facebook.presto.spi.StandardWarningCode.SAMPLED_FIELDS; @@ -148,9 +150,9 @@ private PlanNode addSamplingFilter(PlanNode tableScanNode, Optional ALLOWED_FUNCTIONS = ImmutableSet.of( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "max"), - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "min"), - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "approx_distinct")); - - // Min/Max could be folded into LEAST/GREATEST - private static final Map AGGREGATION_SCALAR_MAPPING = ImmutableMap.of( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "max"), QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "greatest"), - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "min"), QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "least")); - + private final Set allowedFunctions; + private final Map aggregationScalarMapping; private final Metadata metadata; public MetadataQueryOptimizer(Metadata metadata) @@ -95,6 +87,15 @@ public MetadataQueryOptimizer(Metadata metadata) requireNonNull(metadata, "metadata is null"); this.metadata = metadata; + CatalogSchemaName defaultNamespace = metadata.getFunctionAndTypeManager().getDefaultNamespace(); + this.allowedFunctions = ImmutableSet.of( + QualifiedObjectName.valueOf(defaultNamespace, "max"), + QualifiedObjectName.valueOf(defaultNamespace, "min"), + QualifiedObjectName.valueOf(defaultNamespace, "approx_distinct")); + // Min/Max could be folded into LEAST/GREATEST + this.aggregationScalarMapping = ImmutableMap.of( + QualifiedObjectName.valueOf(defaultNamespace, "max"), QualifiedObjectName.valueOf(defaultNamespace, "greatest"), + QualifiedObjectName.valueOf(defaultNamespace, "min"), QualifiedObjectName.valueOf(defaultNamespace, "least")); } @Override @@ -108,6 +109,16 @@ public PlanOptimizerResult optimize(PlanNode plan, Session session, TypeProvider return PlanOptimizerResult.optimizerResult(rewrittenPlan, optimizer.isPlanChanged()); } + public Set getAllowedFunctions() + { + return allowedFunctions; + } + + public Map getAggregationScalarMapping() + { + return aggregationScalarMapping; + } + private static class Optimizer extends SimplePlanRewriter { @@ -118,6 +129,7 @@ private static class Optimizer private final boolean ignoreMetadataStats; private final int metastoreCallNumThreshold; private boolean planChanged; + private final MetadataQueryOptimizer metadataQueryOptimizer; private Optimizer(Session session, Metadata metadata, PlanNodeIdAllocator idAllocator) { @@ -127,6 +139,7 @@ private Optimizer(Session session, Metadata metadata, PlanNodeIdAllocator idAllo this.determinismEvaluator = new RowExpressionDeterminismEvaluator(metadata); this.ignoreMetadataStats = SystemSessionProperties.isOptimizeMetadataQueriesIgnoreStats(session); this.metastoreCallNumThreshold = SystemSessionProperties.getOptimizeMetadataQueriesCallThreshold(session); + this.metadataQueryOptimizer = new MetadataQueryOptimizer(metadata); } public boolean isPlanChanged() @@ -140,7 +153,7 @@ public PlanNode visitAggregation(AggregationNode node, RewriteContext cont // supported functions are only MIN/MAX/APPROX_DISTINCT or distinct aggregates for (Aggregation aggregation : node.getAggregations().values()) { QualifiedObjectName functionName = metadata.getFunctionAndTypeManager().getFunctionMetadata(aggregation.getFunctionHandle()).getName(); - if (!ALLOWED_FUNCTIONS.contains(functionName) && !aggregation.isDistinct()) { + if (!metadataQueryOptimizer.getAllowedFunctions().contains(functionName) && !aggregation.isDistinct()) { return context.defaultRewrite(node); } } @@ -261,7 +274,7 @@ private boolean isReducible(AggregationNode node, List 1 || !inputs.containsAll(aggregation.getCall().getArguments())) { return false; @@ -355,7 +368,7 @@ private RowExpression evaluateMinMax(FunctionMetadata aggregationFunctionMetadat return constant(null, returnType); } - String scalarFunctionName = AGGREGATION_SCALAR_MAPPING.get(aggregationFunctionMetadata.getName()).getObjectName(); + String scalarFunctionName = metadataQueryOptimizer.getAggregationScalarMapping().get(aggregationFunctionMetadata.getName()).getObjectName(); ConnectorSession connectorSession = session.toConnectorSession(); while (arguments.size() > 1) { List reducedArguments = new ArrayList<>(); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java index b3db2189fbfb9..2cb459394a84c 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/OptimizeMixedDistinctAggregations.java @@ -51,7 +51,7 @@ import static com.facebook.presto.common.function.OperatorType.EQUAL; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.spi.plan.AggregationNode.Step.SINGLE; import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.spi.plan.ProjectNode.Locality.LOCAL; @@ -215,9 +215,9 @@ public PlanNode visitAggregation(AggregationNode node, RewriteContext arguments) + { + FunctionHandle functionHandle = functionAndTypeManager.lookupFunction(qualifiedObjectName, fromTypes(arguments.stream().map(RowExpression::getType).collect(toImmutableList()))); + return call(String.valueOf(qualifiedObjectName), functionHandle, returnType, arguments); + } + public static CallExpression call(FunctionAndTypeResolver functionAndTypeResolver, String name, Type returnType, RowExpression... arguments) { FunctionHandle functionHandle = functionAndTypeResolver.lookupFunction(name, fromTypes(Arrays.stream(arguments).map(RowExpression::getType).collect(toImmutableList()))); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java index b0ad53dd7f0db..860d1e5ee3927 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/FunctionResolution.java @@ -17,11 +17,14 @@ import com.facebook.presto.common.function.OperatorType; import com.facebook.presto.common.type.CharType; import com.facebook.presto.common.type.Type; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.StandardErrorCode; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.sql.analyzer.FunctionAndTypeResolver; import com.facebook.presto.sql.tree.ArithmeticBinaryExpression; import com.facebook.presto.sql.tree.ComparisonExpression; +import com.facebook.presto.sql.tree.QualifiedName; import com.google.common.collect.ImmutableList; import java.util.List; @@ -44,7 +47,7 @@ import static com.facebook.presto.common.function.OperatorType.SUBTRACT; import static com.facebook.presto.common.type.BooleanType.BOOLEAN; import static com.facebook.presto.common.type.VarcharType.VARCHAR; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; import static com.facebook.presto.sql.tree.ArrayConstructor.ARRAY_CONSTRUCTOR; import static com.facebook.presto.type.LikePatternType.LIKE_PATTERN; @@ -56,16 +59,17 @@ public final class FunctionResolution implements StandardFunctionResolution { private final FunctionAndTypeResolver functionAndTypeResolver; - private final List windowValueFunctions = ImmutableList.of( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "lead"), - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "lag"), - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "first_value"), - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "last_value"), - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "nth_value")); + private final List windowValueFunctions; public FunctionResolution(FunctionAndTypeResolver functionAndTypeResolver) { this.functionAndTypeResolver = requireNonNull(functionAndTypeResolver, "functionManager is null"); + this.windowValueFunctions = ImmutableList.of( + functionAndTypeResolver.qualifyObjectName(QualifiedName.of("lead")), + functionAndTypeResolver.qualifyObjectName(QualifiedName.of("lag")), + functionAndTypeResolver.qualifyObjectName(QualifiedName.of("first_value")), + functionAndTypeResolver.qualifyObjectName(QualifiedName.of("last_value")), + functionAndTypeResolver.qualifyObjectName(QualifiedName.of("nth_value"))); } @Override @@ -85,6 +89,30 @@ public FunctionHandle likeVarcharFunction() return functionAndTypeResolver.lookupFunction("LIKE", fromTypes(VARCHAR, LIKE_PATTERN)); } + public boolean supportsLikePatternFunction() + { + try { + functionAndTypeResolver.lookupFunction("LIKE_PATTERN", fromTypes(VARCHAR, VARCHAR)); + return true; + } + catch (PrestoException e) { + if (e.getErrorCode() == StandardErrorCode.FUNCTION_NOT_FOUND.toErrorCode()) { + return false; + } + throw e; + } + } + + public FunctionHandle likeVarcharVarcharFunction() + { + return functionAndTypeResolver.lookupFunction("LIKE", fromTypes(VARCHAR, VARCHAR)); + } + + public FunctionHandle likeVarcharVarcharVarcharFunction() + { + return functionAndTypeResolver.lookupFunction("LIKE", fromTypes(VARCHAR, VARCHAR, VARCHAR)); + } + @Override public FunctionHandle likeCharFunction(Type valueType) { @@ -95,7 +123,7 @@ public FunctionHandle likeCharFunction(Type valueType) @Override public boolean isLikeFunction(FunctionHandle functionHandle) { - return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "LIKE")); + return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "LIKE")); } @Override @@ -107,7 +135,11 @@ public FunctionHandle likePatternFunction() @Override public boolean isLikePatternFunction(FunctionHandle functionHandle) { - return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "LIKE_PATTERN")); + QualifiedObjectName name = + supportsLikePatternFunction() ? + functionAndTypeResolver.qualifyObjectName(QualifiedName.of("LIKE_PATTERN")) : + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "LIKE_PATTERN"); + return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(name); } @Override @@ -118,12 +150,12 @@ public boolean isCastFunction(FunctionHandle functionHandle) public boolean isTryCastFunction(FunctionHandle functionHandle) { - return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "TRY_CAST")); + return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "TRY_CAST")); } public boolean isArrayConstructor(FunctionHandle functionHandle) { - return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, ARRAY_CONSTRUCTOR)); + return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(functionAndTypeResolver.qualifyObjectName(QualifiedName.of(ARRAY_CONSTRUCTOR))); } @Override @@ -271,13 +303,13 @@ public boolean isTryFunction(FunctionHandle functionHandle) public boolean isFailFunction(FunctionHandle functionHandle) { - return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "fail")); + return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(functionAndTypeResolver.qualifyObjectName(QualifiedName.of("fail"))); } @Override public boolean isCountFunction(FunctionHandle functionHandle) { - return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "count")); + return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(functionAndTypeResolver.qualifyObjectName(QualifiedName.of("count"))); } @Override @@ -295,7 +327,7 @@ public FunctionHandle countFunction(Type valueType) @Override public boolean isMaxFunction(FunctionHandle functionHandle) { - return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "max")); + return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(functionAndTypeResolver.qualifyObjectName(QualifiedName.of("max"))); } @Override @@ -313,7 +345,7 @@ public FunctionHandle greatestFunction(List valueTypes) @Override public boolean isMinFunction(FunctionHandle functionHandle) { - return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "min")); + return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(functionAndTypeResolver.qualifyObjectName(QualifiedName.of("min"))); } @Override @@ -331,7 +363,7 @@ public FunctionHandle leastFunction(List valueTypes) @Override public boolean isApproximateCountDistinctFunction(FunctionHandle functionHandle) { - return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "approx_distinct")); + return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(functionAndTypeResolver.qualifyObjectName(QualifiedName.of("approx_distinct"))); } @Override @@ -343,7 +375,7 @@ public FunctionHandle approximateCountDistinctFunction(Type valueType) @Override public boolean isApproximateSetFunction(FunctionHandle functionHandle) { - return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "approx_set")); + return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(functionAndTypeResolver.qualifyObjectName(QualifiedName.of("approx_set"))); } @Override @@ -359,12 +391,12 @@ public boolean isEqualFunction(FunctionHandle functionHandle) public boolean isArrayContainsFunction(FunctionHandle functionHandle) { - return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "contains")); + return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(functionAndTypeResolver.qualifyObjectName(QualifiedName.of("contains"))); } public boolean isElementAtFunction(FunctionHandle functionHandle) { - return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "element_at")); + return functionAndTypeResolver.getFunctionMetadata(functionHandle).getName().equals(functionAndTypeResolver.qualifyObjectName(QualifiedName.of("element_at"))); } public boolean isWindowValueFunction(FunctionHandle functionHandle) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java b/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java index a39180396d2a4..c33430616a779 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/relational/SqlToRowExpressionTranslator.java @@ -908,6 +908,9 @@ protected RowExpression visitLikePredicate(LikePredicate node, Context context) if (node.getEscape().isPresent()) { RowExpression escape = process(node.getEscape().get(), context); + if (!functionResolution.supportsLikePatternFunction()) { + return call(value.getSourceLocation(), "LIKE", functionResolution.likeVarcharVarcharVarcharFunction(), BOOLEAN, value, pattern, escape); + } return likeFunctionCall(value, call(getSourceLocation(node), "LIKE_PATTERN", functionResolution.likePatternFunction(), LIKE_PATTERN, pattern, escape)); } @@ -916,6 +919,10 @@ protected RowExpression visitLikePredicate(LikePredicate node, Context context) return prefixOrSuffixMatch; } + if (!functionResolution.supportsLikePatternFunction()) { + return likeFunctionCall(value, pattern); + } + return likeFunctionCall(value, call(getSourceLocation(node), CAST.name(), functionAndTypeResolver.lookupCast("CAST", VARCHAR, LIKE_PATTERN), LIKE_PATTERN, pattern)); } @@ -961,6 +968,9 @@ else if (LIKE_SIMPLE_EXISTS_PATTERN.matcher(patternString).matches()) { private RowExpression likeFunctionCall(RowExpression value, RowExpression pattern) { if (value.getType() instanceof VarcharType) { + if (!functionResolution.supportsLikePatternFunction()) { + return call(value.getSourceLocation(), "LIKE", functionResolution.likeVarcharVarcharFunction(), BOOLEAN, value, pattern); + } return call(value.getSourceLocation(), "LIKE", functionResolution.likeVarcharFunction(), BOOLEAN, value, pattern); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/rewrite/ShowQueriesRewrite.java b/presto-main/src/main/java/com/facebook/presto/sql/rewrite/ShowQueriesRewrite.java index 0b56b19409727..a719c9357baaf 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/rewrite/ShowQueriesRewrite.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/rewrite/ShowQueriesRewrite.java @@ -107,7 +107,7 @@ import static com.facebook.presto.connector.informationSchema.InformationSchemaMetadata.TABLE_SCHEMATA; import static com.facebook.presto.connector.informationSchema.InformationSchemaMetadata.TABLE_TABLES; import static com.facebook.presto.connector.informationSchema.InformationSchemaMetadata.TABLE_TABLE_PRIVILEGES; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.metadata.MetadataListing.listCatalogs; import static com.facebook.presto.metadata.MetadataListing.listSchemas; import static com.facebook.presto.metadata.MetadataUtil.createCatalogSchemaName; @@ -696,10 +696,12 @@ protected Node visitShowFunctions(ShowFunctions node, Void context) for (SqlFunction function : metadata.getFunctionAndTypeManager().listFunctions(session, node.getLikePattern(), node.getEscape())) { Signature signature = function.getSignature(); - boolean builtIn = signature.getName().getCatalogSchemaName().equals(DEFAULT_NAMESPACE); + boolean builtIn = signature.getName().getCatalogSchemaName().equals(JAVA_BUILTIN_NAMESPACE); boolean temporary = signature.getName().getCatalogSchemaName().equals(SESSION_NAMESPACE); + boolean current = signature.getName().getCatalogSchemaName().equals( + metadata.getFunctionAndTypeManager().getDefaultNamespace()); rows.add(row( - builtIn || temporary ? new StringLiteral(signature.getNameSuffix()) : new StringLiteral(signature.getName().toString()), + builtIn || temporary || current ? new StringLiteral(signature.getNameSuffix()) : new StringLiteral(signature.getName().toString()), new StringLiteral(signature.getReturnType().toString()), new StringLiteral(Joiner.on(", ").join(signature.getArgumentTypes())), new StringLiteral(getFunctionType(function)), diff --git a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java index a01b7af8d57a4..8d2a9ac22e5bb 100644 --- a/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java +++ b/presto-main/src/main/java/com/facebook/presto/testing/LocalQueryRunner.java @@ -130,6 +130,7 @@ import com.facebook.presto.server.security.PrestoAuthenticatorManager; import com.facebook.presto.server.security.SecurityConfig; import com.facebook.presto.spi.ConnectorId; +import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.PageIndexerFactory; import com.facebook.presto.spi.PageSorter; import com.facebook.presto.spi.Plugin; @@ -356,6 +357,7 @@ public class LocalQueryRunner private final PlanChecker distributedPlanChecker; private final PlanChecker singleNodePlanChecker; + private final NodeManager pluginNodeManager; private static ExecutorService metadataExtractorExecutor = newCachedThreadPool(threadsNamed("query-execution-%s")); @@ -477,6 +479,7 @@ private LocalQueryRunner(Session defaultSession, FeaturesConfig featuresConfig, this.pageFunctionCompiler = new PageFunctionCompiler(metadata, 0); this.expressionCompiler = new ExpressionCompiler(metadata, pageFunctionCompiler); this.joinFilterFunctionCompiler = new JoinFilterFunctionCompiler(metadata); + this.pluginNodeManager = new PluginNodeManager(nodeManager, "test"); NodeVersion nodeVersion = new NodeVersion("testversion"); this.connectorManager = new ConnectorManager( @@ -772,7 +775,7 @@ public void createCatalog(String catalogName, String connectorName, Map properties) { - metadata.getFunctionAndTypeManager().loadFunctionNamespaceManager(functionNamespaceManagerName, catalogName, properties); + metadata.getFunctionAndTypeManager().loadFunctionNamespaceManager(functionNamespaceManagerName, catalogName, properties, pluginNodeManager); } public LocalQueryRunner printPlan() diff --git a/presto-main/src/main/java/com/facebook/presto/util/SpatialJoinUtils.java b/presto-main/src/main/java/com/facebook/presto/util/SpatialJoinUtils.java index adb4bd5fa1dff..fed1e2598c29c 100644 --- a/presto-main/src/main/java/com/facebook/presto/util/SpatialJoinUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/util/SpatialJoinUtils.java @@ -37,7 +37,7 @@ import static com.facebook.presto.common.function.OperatorType.LESS_THAN; import static com.facebook.presto.common.function.OperatorType.LESS_THAN_OR_EQUAL; import static com.facebook.presto.common.function.OperatorType.NOT_EQUAL; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -45,14 +45,14 @@ public class SpatialJoinUtils { - public static final QualifiedObjectName ST_CONTAINS = QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "st_contains"); - public static final QualifiedObjectName ST_CROSSES = QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "st_crosses"); - public static final QualifiedObjectName ST_EQUALS = QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "st_equals"); - public static final QualifiedObjectName ST_INTERSECTS = QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "st_intersects"); - public static final QualifiedObjectName ST_OVERLAPS = QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "st_overlaps"); - public static final QualifiedObjectName ST_TOUCHES = QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "st_touches"); - public static final QualifiedObjectName ST_WITHIN = QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "st_within"); - public static final QualifiedObjectName ST_DISTANCE = QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "st_distance"); + public static final QualifiedObjectName ST_CONTAINS = QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "st_contains"); + public static final QualifiedObjectName ST_CROSSES = QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "st_crosses"); + public static final QualifiedObjectName ST_EQUALS = QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "st_equals"); + public static final QualifiedObjectName ST_INTERSECTS = QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "st_intersects"); + public static final QualifiedObjectName ST_OVERLAPS = QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "st_overlaps"); + public static final QualifiedObjectName ST_TOUCHES = QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "st_touches"); + public static final QualifiedObjectName ST_WITHIN = QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "st_within"); + public static final QualifiedObjectName ST_DISTANCE = QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "st_distance"); private static final Set ALLOWED_SPATIAL_JOIN_FUNCTIONS = Stream.of( ST_CONTAINS, ST_CROSSES, ST_EQUALS, ST_INTERSECTS, ST_OVERLAPS, ST_TOUCHES, ST_WITHIN) diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/TestSignature.java b/presto-main/src/test/java/com/facebook/presto/metadata/TestSignature.java index 929af98936555..d46e7d74363af 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/TestSignature.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/TestSignature.java @@ -26,7 +26,7 @@ import org.testng.annotations.Test; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.metadata.FunctionAndTypeManager.createTestFunctionAndTypeManager; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; import static org.testng.Assert.assertEquals; @@ -41,7 +41,7 @@ public void testSerializationRoundTrip() JsonCodec codec = new JsonCodecFactory(objectMapperProvider, true).jsonCodec(Signature.class); Signature expected = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "function"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "function"), SCALAR, parseTypeSignature(StandardTypes.BIGINT), ImmutableList.of(parseTypeSignature(StandardTypes.BOOLEAN), parseTypeSignature(StandardTypes.DOUBLE), parseTypeSignature(StandardTypes.VARCHAR))); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/GenericLongFunction.java b/presto-main/src/test/java/com/facebook/presto/operator/GenericLongFunction.java index 05d9fd1383099..3549ebb7706a3 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/GenericLongFunction.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/GenericLongFunction.java @@ -27,7 +27,7 @@ import static com.facebook.presto.common.type.StandardTypes.BIGINT; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; @@ -46,7 +46,7 @@ public final class GenericLongFunction GenericLongFunction(String suffix, LongUnaryOperator longUnaryOperator) { - super(new Signature(QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "generic_long_" + requireNonNull(suffix, "suffix is null")), SCALAR, + super(new Signature(QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "generic_long_" + requireNonNull(suffix, "suffix is null")), SCALAR, emptyList(), emptyList(), parseTypeSignature(BIGINT), singletonList(parseTypeSignature(BIGINT)), false)); this.longUnaryOperator = longUnaryOperator; } diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForAggregates.java b/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForAggregates.java index 6701e961e29ae..5927f1fc65ee6 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForAggregates.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForAggregates.java @@ -66,7 +66,7 @@ import static com.facebook.presto.common.type.StandardTypes.ARRAY; import static com.facebook.presto.common.type.StandardTypes.DOUBLE; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.metadata.FunctionAndTypeManager.createTestFunctionAndTypeManager; import static com.facebook.presto.operator.aggregation.AggregationFromAnnotationsParser.parseFunctionDefinition; import static com.facebook.presto.operator.aggregation.AggregationFromAnnotationsParser.parseFunctionDefinitions; @@ -109,7 +109,7 @@ public static void output(@AggregationState NullableDoubleState state, BlockBuil public void testSimpleExactAggregationParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "simple_exact_aggregate"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "simple_exact_aggregate"), FunctionKind.AGGREGATE, DoubleType.DOUBLE.getTypeSignature(), ImmutableList.of(DoubleType.DOUBLE.getTypeSignature())); @@ -161,7 +161,7 @@ public static void output(BlockBuilder out, @AggregationState NullableDoubleStat public void testStateOnDifferentThanFirstPositionAggregationParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "simple_exact_aggregate_aggregation_state_moved"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "simple_exact_aggregate_aggregation_state_moved"), FunctionKind.AGGREGATE, DoubleType.DOUBLE.getTypeSignature(), ImmutableList.of(DoubleType.DOUBLE.getTypeSignature())); @@ -303,7 +303,7 @@ public static AccumulatorStateSerializer createSerializer() public void testNotDecomposableAggregationParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "custom_decomposable_aggregate"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "custom_decomposable_aggregate"), FunctionKind.AGGREGATE, DoubleType.DOUBLE.getTypeSignature(), ImmutableList.of(DoubleType.DOUBLE.getTypeSignature())); @@ -378,7 +378,7 @@ public static void output( public void testSimpleGenericAggregationFunctionParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "simple_generic_implementations"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "simple_generic_implementations"), FunctionKind.AGGREGATE, ImmutableList.of(typeVariable("T")), ImmutableList.of(), @@ -453,7 +453,7 @@ public static void output( public void testSimpleBlockInputAggregationParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "block_input_aggregate"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "block_input_aggregate"), FunctionKind.AGGREGATE, DoubleType.DOUBLE.getTypeSignature(), ImmutableList.of(DoubleType.DOUBLE.getTypeSignature())); @@ -537,7 +537,7 @@ public static void output( public void testSimpleImplicitSpecializedAggregationParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "implicit_specialized_aggregate"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "implicit_specialized_aggregate"), FunctionKind.AGGREGATE, ImmutableList.of(typeVariable("T")), ImmutableList.of(), @@ -629,7 +629,7 @@ public static void output( public void testSimpleExplicitSpecializedAggregationParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "explicit_specialized_aggregate"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "explicit_specialized_aggregate"), FunctionKind.AGGREGATE, ImmutableList.of(typeVariable("T")), ImmutableList.of(), @@ -704,13 +704,13 @@ public static void output2( public void testMultiOutputAggregationParse() { Signature expectedSignature1 = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "multi_output_aggregate_1"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "multi_output_aggregate_1"), FunctionKind.AGGREGATE, DoubleType.DOUBLE.getTypeSignature(), ImmutableList.of(DoubleType.DOUBLE.getTypeSignature())); Signature expectedSignature2 = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "multi_output_aggregate_2"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "multi_output_aggregate_2"), FunctionKind.AGGREGATE, DoubleType.DOUBLE.getTypeSignature(), ImmutableList.of(DoubleType.DOUBLE.getTypeSignature())); @@ -790,7 +790,7 @@ public static CustomStateSerializerAggregationFunction.CustomSerializer createSe public void testInjectOperatorAggregateParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "inject_operator_aggregate"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "inject_operator_aggregate"), FunctionKind.AGGREGATE, DoubleType.DOUBLE.getTypeSignature(), ImmutableList.of(DoubleType.DOUBLE.getTypeSignature())); @@ -868,7 +868,7 @@ public static CustomStateSerializerAggregationFunction.CustomSerializer createSe public void testInjectTypeAggregateParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "inject_type_aggregate"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "inject_type_aggregate"), FunctionKind.AGGREGATE, ImmutableList.of(typeVariable("T")), ImmutableList.of(), @@ -950,7 +950,7 @@ public static CustomStateSerializerAggregationFunction.CustomSerializer createSe public void testInjectLiteralAggregateParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "inject_literal_aggregate"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "inject_literal_aggregate"), FunctionKind.AGGREGATE, parseTypeSignature("varchar(x)", ImmutableSet.of("x")), ImmutableList.of(parseTypeSignature("varchar(x)", ImmutableSet.of("x")))); @@ -1021,7 +1021,7 @@ public static void output( public void testLongConstraintAggregateFunctionParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "parametric_aggregate_long_constraint"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "parametric_aggregate_long_constraint"), FunctionKind.AGGREGATE, ImmutableList.of(), ImmutableList.of(new LongVariableConstraint("z", "x + y")), @@ -1096,7 +1096,7 @@ public static void output( public void testFixedTypeParameterInjectionAggregateFunctionParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "fixed_type_parameter_injection"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "fixed_type_parameter_injection"), FunctionKind.AGGREGATE, ImmutableList.of(), ImmutableList.of(), @@ -1162,7 +1162,7 @@ public static void output( public void testPartiallyFixedTypeParameterInjectionAggregateFunctionParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "partially_fixed_type_parameter_injection"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "partially_fixed_type_parameter_injection"), FunctionKind.AGGREGATE, ImmutableList.of(typeVariable("T1"), typeVariable("T2")), ImmutableList.of(), diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForScalars.java b/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForScalars.java index 2000b520f2c6c..80ab4502de2e6 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForScalars.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForScalars.java @@ -48,7 +48,7 @@ import static com.facebook.presto.common.type.BooleanType.BOOLEAN; import static com.facebook.presto.common.type.DoubleType.DOUBLE; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.metadata.FunctionAndTypeManager.createTestFunctionAndTypeManager; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; @@ -84,7 +84,7 @@ public static double fun(@SqlType(StandardTypes.DOUBLE) double v) public void testSingleImplementationScalarParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "single_implementation_parametric_scalar"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "single_implementation_parametric_scalar"), FunctionKind.SCALAR, DOUBLE.getTypeSignature(), ImmutableList.of(DOUBLE.getTypeSignature())); @@ -190,7 +190,7 @@ public static double fun( public void testWithNullablePrimitiveArgScalarParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "scalar_with_nullable"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "scalar_with_nullable"), FunctionKind.SCALAR, DOUBLE.getTypeSignature(), ImmutableList.of(DOUBLE.getTypeSignature(), DOUBLE.getTypeSignature())); @@ -228,7 +228,7 @@ public static double fun( public void testWithNullableComplexArgScalarParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "scalar_with_nullable_complex"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "scalar_with_nullable_complex"), FunctionKind.SCALAR, DOUBLE.getTypeSignature(), ImmutableList.of(DOUBLE.getTypeSignature(), DOUBLE.getTypeSignature())); @@ -264,7 +264,7 @@ public static double fun(@SqlType(StandardTypes.DOUBLE) double v) public void testStaticMethodScalarParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "static_method_scalar"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "static_method_scalar"), FunctionKind.SCALAR, DOUBLE.getTypeSignature(), ImmutableList.of(DOUBLE.getTypeSignature())); @@ -310,19 +310,19 @@ public static long fun3(@SqlType(StandardTypes.BIGINT) long v) public void testMultiScalarParse() { Signature expectedSignature1 = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "static_method_scalar_1"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "static_method_scalar_1"), FunctionKind.SCALAR, DOUBLE.getTypeSignature(), ImmutableList.of(DOUBLE.getTypeSignature())); Signature expectedSignature2 = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "static_method_scalar_2"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "static_method_scalar_2"), FunctionKind.SCALAR, BIGINT.getTypeSignature(), ImmutableList.of(BIGINT.getTypeSignature())); Signature expectedSignature3 = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "static_method_scalar_3"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "static_method_scalar_3"), FunctionKind.SCALAR, BIGINT.getTypeSignature(), ImmutableList.of(BIGINT.getTypeSignature())); @@ -375,7 +375,7 @@ public static long fun(@SqlType("T") long v) public void testParametricScalarParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "parametric_scalar"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "parametric_scalar"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T")), ImmutableList.of(), @@ -416,7 +416,7 @@ public static boolean fun2(@SqlType("array(varchar(17))") Block array) public void testComplexParametricScalarParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "with_exact_scalar"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "with_exact_scalar"), FunctionKind.SCALAR, ImmutableList.of(), ImmutableList.of(), @@ -425,7 +425,7 @@ public void testComplexParametricScalarParse() false); Signature exactSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "with_exact_scalar"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "with_exact_scalar"), FunctionKind.SCALAR, ImmutableList.of(), ImmutableList.of(), @@ -463,7 +463,7 @@ public static long fun( public void testSimpleInjectionScalarParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "parametric_scalar_inject"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "parametric_scalar_inject"), FunctionKind.SCALAR, ImmutableList.of(), ImmutableList.of(), @@ -523,7 +523,7 @@ public long funDouble(@SqlType("array(double)") Block val) public void testConstructorInjectionScalarParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "parametric_scalar_inject_constructor"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "parametric_scalar_inject_constructor"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T")), ImmutableList.of(), @@ -566,7 +566,7 @@ public static long fun( public void testFixedTypeParameterParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "fixed_type_parameter_scalar_function"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "fixed_type_parameter_scalar_function"), FunctionKind.SCALAR, ImmutableList.of(), ImmutableList.of(), @@ -604,7 +604,7 @@ public static long fun( public void testPartiallyFixedTypeParameterParse() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "partially_fixed_type_parameter_scalar_function"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "partially_fixed_type_parameter_scalar_function"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T1"), typeVariable("T2")), ImmutableList.of(), diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForSqlInvokedScalars.java b/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForSqlInvokedScalars.java index 0a545d8e58dd4..053635444a321 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForSqlInvokedScalars.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestAnnotationEngineForSqlInvokedScalars.java @@ -34,7 +34,7 @@ import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.common.type.IntegerType.INTEGER; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; @@ -46,7 +46,7 @@ public class TestAnnotationEngineForSqlInvokedScalars public void testParseFunctionDefinition() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "sample_sql_invoked_scalar_function"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "sample_sql_invoked_scalar_function"), FunctionKind.SCALAR, new ArrayType(BIGINT).getTypeSignature(), ImmutableList.of(INTEGER.getTypeSignature())); @@ -67,7 +67,7 @@ public void testParseFunctionDefinition() public void testParseFunctionDefinitionWithTypeParameter() { Signature expectedSignature = new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "sample_sql_invoked_scalar_function_with_type_parameter"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "sample_sql_invoked_scalar_function_with_type_parameter"), FunctionKind.SCALAR, ImmutableList.of(new TypeVariableConstraint("T", false, false, null, false)), Collections.emptyList(), diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayFilter.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayFilter.java index 99a50fab6439d..f8dc2987a0bbf 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayFilter.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/BenchmarkArrayFilter.java @@ -67,7 +67,7 @@ import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.common.type.TypeUtils.readNativeValue; import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.BenchmarkArrayFilter.ExactArrayFilterFunction.EXACT_ARRAY_FILTER_FUNCTION; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; @@ -205,7 +205,7 @@ public static final class ExactArrayFilterFunction private ExactArrayFilterFunction() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "exact_filter"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "exact_filter"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T")), ImmutableList.of(), diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestProvidedBlockBuilderReturnPlaceConvention.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestProvidedBlockBuilderReturnPlaceConvention.java index 99f0fd473619b..25f01aa8be0f0 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestProvidedBlockBuilderReturnPlaceConvention.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/TestProvidedBlockBuilderReturnPlaceConvention.java @@ -41,7 +41,7 @@ import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.common.type.VarcharType.VARCHAR; import static com.facebook.presto.common.type.VarcharType.createVarcharType; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.USE_BOXED_TYPE; @@ -155,7 +155,7 @@ public static class FunctionWithProvidedBlockReturnPlaceConvention1 protected FunctionWithProvidedBlockReturnPlaceConvention1() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "identity1"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "identity1"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T")), ImmutableList.of(), @@ -275,7 +275,7 @@ public static class FunctionWithProvidedBlockReturnPlaceConvention2 protected FunctionWithProvidedBlockReturnPlaceConvention2() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "identity2"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "identity2"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("T")), ImmutableList.of(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java b/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java index 5f751b22d10e2..f07caf36cf5e8 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/TestExpressionInterpreter.java @@ -59,6 +59,7 @@ import com.facebook.presto.sql.tree.NodeRef; import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.StringLiteral; +import com.facebook.presto.testing.TestingNodeManager; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -433,7 +434,8 @@ private void setupJsonFunctionNamespaceManager(FunctionAndTypeManager functionAn functionAndTypeManager.loadFunctionNamespaceManager( JsonFileBasedFunctionNamespaceManagerFactory.NAME, "json", - ImmutableMap.of("supported-function-languages", "CPP", "function-implementation-type", "CPP")); + ImmutableMap.of("supported-function-languages", "CPP", "function-implementation-type", "CPP"), + new TestingNodeManager()); } @Test diff --git a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFunctionsConfig.java b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFunctionsConfig.java index 445bb6c819b18..e5f8cd752e2ab 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFunctionsConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFunctionsConfig.java @@ -24,6 +24,7 @@ import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.sql.analyzer.RegexLibrary.JONI; public class TestFunctionsConfig @@ -52,7 +53,8 @@ public void testDefaults() .setFieldNamesInJsonCastEnabled(false) .setWarnOnCommonNanPatterns(false) .setLegacyCharToVarcharCoercion(false) - .setLegacyJsonCast(true)); + .setLegacyJsonCast(true) + .setDefaultNamespacePrefix(JAVA_BUILTIN_NAMESPACE.toString())); } @Test @@ -80,6 +82,7 @@ public void testExplicitPropertyMappings() .put("warn-on-common-nan-patterns", "true") .put("deprecated.legacy-char-to-varchar-coercion", "true") .put("legacy-json-cast", "false") + .put("presto.default-namespace", "native.default") .build(); FunctionsConfig expected = new FunctionsConfig() @@ -103,7 +106,8 @@ public void testExplicitPropertyMappings() .setFieldNamesInJsonCastEnabled(true) .setWarnOnCommonNanPatterns(true) .setLegacyCharToVarcharCoercion(true) - .setLegacyJsonCast(false); + .setLegacyJsonCast(false) + .setDefaultNamespacePrefix("native.default"); assertFullMapping(properties, expected); } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/gen/TestVarArgsToArrayAdapterGenerator.java b/presto-main/src/test/java/com/facebook/presto/sql/gen/TestVarArgsToArrayAdapterGenerator.java index aa2271b624033..29f13d5e26f46 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/gen/TestVarArgsToArrayAdapterGenerator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/gen/TestVarArgsToArrayAdapterGenerator.java @@ -32,7 +32,7 @@ import java.util.stream.IntStream; import static com.facebook.presto.common.type.IntegerType.INTEGER; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.SqlFunctionVisibility.PUBLIC; @@ -77,7 +77,7 @@ public static class TestVarArgsSum private TestVarArgsSum() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "var_args_sum"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "var_args_sum"), FunctionKind.SCALAR, ImmutableList.of(), ImmutableList.of(), diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionVariableInliner.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionVariableInliner.java index f3fd696b9ded3..1c9d9732a232f 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionVariableInliner.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestRowExpressionVariableInliner.java @@ -15,6 +15,7 @@ import com.facebook.presto.common.CatalogSchemaName; import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.spi.function.FunctionHandle; import com.facebook.presto.spi.function.FunctionKind; import com.facebook.presto.spi.relation.CallExpression; @@ -24,10 +25,12 @@ import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; +import java.util.List; import java.util.Optional; import static com.facebook.presto.common.type.BigintType.BIGINT; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; +import static java.util.Collections.emptyList; import static org.testng.Assert.assertEquals; public class TestRowExpressionVariableInliner @@ -52,6 +55,12 @@ public FunctionKind getKind() { return SCALAR; } + + @Override + public List getArgumentTypes() + { + return emptyList(); + } } private static final FunctionHandle TEST_FUNCTION = new TestFunctionHandle(); diff --git a/presto-native-execution/presto_cpp/main/PrestoServer.cpp b/presto-native-execution/presto_cpp/main/PrestoServer.cpp index f79f1b5ee1fce..0a422c66e16bd 100644 --- a/presto-native-execution/presto_cpp/main/PrestoServer.cpp +++ b/presto-native-execution/presto_cpp/main/PrestoServer.cpp @@ -241,6 +241,7 @@ void PrestoServer::run() { address_ = fmt::format("[{}]", address_); } nodeLocation_ = nodeConfig->nodeLocation(); + prestoBuiltinFunctionPrefix_ = systemConfig->prestoDefaultNamespacePrefix(); } catch (const velox::VeloxUserError& e) { PRESTO_STARTUP_LOG(ERROR) << "Failed to start server due to " << e.what(); exit(EXIT_FAILURE); @@ -1308,11 +1309,12 @@ void PrestoServer::registerCustomOperators() { } void PrestoServer::registerFunctions() { - static const std::string kPrestoDefaultPrefix{"presto.default."}; - velox::functions::prestosql::registerAllScalarFunctions(kPrestoDefaultPrefix); + velox::functions::prestosql::registerAllScalarFunctions( + prestoBuiltinFunctionPrefix_); velox::aggregate::prestosql::registerAllAggregateFunctions( - kPrestoDefaultPrefix); - velox::window::prestosql::registerAllWindowFunctions(kPrestoDefaultPrefix); + prestoBuiltinFunctionPrefix_); + velox::window::prestosql::registerAllWindowFunctions( + prestoBuiltinFunctionPrefix_); if (SystemConfig::instance()->registerTestFunctions()) { velox::functions::prestosql::registerAllScalarFunctions( "json.test_schema."); diff --git a/presto-native-execution/presto_cpp/main/PrestoServer.h b/presto-native-execution/presto_cpp/main/PrestoServer.h index 615d6bfb910d9..9fa1301c1bf1d 100644 --- a/presto-native-execution/presto_cpp/main/PrestoServer.h +++ b/presto-native-execution/presto_cpp/main/PrestoServer.h @@ -291,6 +291,7 @@ class PrestoServer { std::string address_; std::string nodeLocation_; folly::SSLContextPtr sslContext_; + std::string prestoBuiltinFunctionPrefix_; }; } // namespace facebook::presto diff --git a/presto-native-execution/presto_cpp/main/common/Configs.cpp b/presto-native-execution/presto_cpp/main/common/Configs.cpp index 35009adc1cba7..2ef3d8253ab3d 100644 --- a/presto-native-execution/presto_cpp/main/common/Configs.cpp +++ b/presto-native-execution/presto_cpp/main/common/Configs.cpp @@ -239,6 +239,7 @@ SystemConfig::SystemConfig() { STR_PROP(kCacheVeloxTtlCheckInterval, "1h"), BOOL_PROP(kEnableRuntimeMetricsCollection, false), BOOL_PROP(kPlanValidatorFailOnNestedLoopJoin, false), + STR_PROP(kPrestoDefaultNamespacePrefix, "presto.default"), }; } @@ -758,6 +759,10 @@ bool SystemConfig::enableRuntimeMetricsCollection() const { return optionalProperty(kEnableRuntimeMetricsCollection).value(); } +std::string SystemConfig::prestoDefaultNamespacePrefix() const { + return optionalProperty(kPrestoDefaultNamespacePrefix).value().append("."); +} + NodeConfig::NodeConfig() { registeredProps_ = std::unordered_map>{ diff --git a/presto-native-execution/presto_cpp/main/common/Configs.h b/presto-native-execution/presto_cpp/main/common/Configs.h index fe2fa4e68f807..5969006e62575 100644 --- a/presto-native-execution/presto_cpp/main/common/Configs.h +++ b/presto-native-execution/presto_cpp/main/common/Configs.h @@ -655,6 +655,10 @@ class SystemConfig : public ConfigBase { static constexpr std::string_view kPlanValidatorFailOnNestedLoopJoin{ "velox-plan-validator-fail-on-nested-loop-join"}; + // Specifies the default Presto namespace prefix. + static constexpr std::string_view kPrestoDefaultNamespacePrefix{ + "presto.default-namespace"}; + SystemConfig(); virtual ~SystemConfig() = default; @@ -893,6 +897,7 @@ class SystemConfig : public ConfigBase { bool enableRuntimeMetricsCollection() const; bool prestoNativeSidecar() const; + std::string prestoDefaultNamespacePrefix() const; }; /// Provides access to node properties defined in node.properties file. diff --git a/presto-native-execution/presto_cpp/main/common/Utils.h b/presto-native-execution/presto_cpp/main/common/Utils.h index 9c5f3f198f5e2..d481faa0af363 100644 --- a/presto-native-execution/presto_cpp/main/common/Utils.h +++ b/presto-native-execution/presto_cpp/main/common/Utils.h @@ -47,4 +47,9 @@ void installSignalHandler(); std::string extractMessageBody( const std::vector>& body); +inline std::string addDefaultNamespacePrefix( + const std::string& prestoDefaultNamespacePrefix, + const std::string& functionName) { + return fmt::format("{}{}", prestoDefaultNamespacePrefix, functionName); +} } // namespace facebook::presto::util diff --git a/presto-native-execution/presto_cpp/main/common/tests/CMakeLists.txt b/presto-native-execution/presto_cpp/main/common/tests/CMakeLists.txt index 3572595147323..1761f70856c82 100644 --- a/presto-native-execution/presto_cpp/main/common/tests/CMakeLists.txt +++ b/presto-native-execution/presto_cpp/main/common/tests/CMakeLists.txt @@ -18,8 +18,13 @@ target_link_libraries( presto_common_test presto_common presto_exception + velox_aggregates velox_exception velox_file + velox_functions_prestosql + velox_function_registry + velox_presto_types + velox_window ${RE2} GTest::gtest GTest::gtest_main) diff --git a/presto-native-execution/presto_cpp/main/common/tests/ConfigTest.cpp b/presto-native-execution/presto_cpp/main/common/tests/ConfigTest.cpp index 9f3282a2308a5..149427cd49ed0 100644 --- a/presto-native-execution/presto_cpp/main/common/tests/ConfigTest.cpp +++ b/presto-native-execution/presto_cpp/main/common/tests/ConfigTest.cpp @@ -12,17 +12,59 @@ * limitations under the License. */ #include +#include #include "presto_cpp/main/common/ConfigReader.h" #include "presto_cpp/main/common/Configs.h" #include "velox/common/base/Exceptions.h" #include "velox/common/base/tests/GTestUtils.h" #include "velox/common/file/File.h" #include "velox/common/file/FileSystems.h" +#include "velox/exec/Aggregate.h" +#include "velox/exec/Window.h" +#include "velox/functions/FunctionRegistry.h" +#include "velox/functions/prestosql/aggregates/RegisterAggregateFunctions.h" +#include "velox/functions/prestosql/registration/RegistrationFunctions.h" +#include "velox/functions/prestosql/window/WindowFunctionsRegistration.h" namespace facebook::presto::test { using namespace velox; +namespace { + +template +bool validateDefaultNamespacePrefix( + const FunctionMap& functionMap, + const std::string& prestoDefaultNamespacePrefix) { + static const std::unordered_set kBlockList = { + "row_constructor", "in", "is_null"}; + + std::vector prestoDefaultNamespacePrefixParts; + folly::split( + '.', + prestoDefaultNamespacePrefix, + prestoDefaultNamespacePrefixParts, + true); + VELOX_CHECK_EQ(prestoDefaultNamespacePrefixParts.size(), 2); + for (const auto& [functionName, _] : functionMap) { + // Skip internal functions. They don't have any prefix. + if ((kBlockList.count(functionName) != 0) || + (functionName.find("$internal$") != std::string::npos)) { + continue; + } + + std::vector parts; + folly::split('.', functionName, parts, true); + VELOX_CHECK_EQ(parts.size(), 3); + if ((parts[0] != prestoDefaultNamespacePrefixParts[0]) || + (parts[1] != prestoDefaultNamespacePrefixParts[1])) { + return false; + } + } + return true; +} +} // namespace + class ConfigTest : public testing::Test { protected: void SetUp() override { @@ -310,4 +352,35 @@ TEST_F(ConfigTest, readConfigEnvVarTest) { unsetenv(kEmptyEnvVarName.c_str()); } +TEST_F(ConfigTest, prestoDefaultNamespacePrefix) { + SystemConfig config; + init( + config, + {{std::string(SystemConfig::kPrestoDefaultNamespacePrefix), + "presto.default"}}); + std::string prestoBuiltinFunctionPrefix = + config.prestoDefaultNamespacePrefix(); + + velox::functions::prestosql::registerAllScalarFunctions( + prestoBuiltinFunctionPrefix); + velox::aggregate::prestosql::registerAllAggregateFunctions( + prestoBuiltinFunctionPrefix); + velox::window::prestosql::registerAllWindowFunctions( + prestoBuiltinFunctionPrefix); + + // Get all registered scalar functions in Velox + auto scalarFunctions = getFunctionSignatures(); + ASSERT_TRUE(validateDefaultNamespacePrefix( + scalarFunctions, prestoBuiltinFunctionPrefix)); + + // Get all registered aggregate functions in Velox + auto aggregateFunctions = exec::aggregateFunctions().copy(); + ASSERT_TRUE(validateDefaultNamespacePrefix( + aggregateFunctions, prestoBuiltinFunctionPrefix)); + + // Get all registered window functions in Velox + auto windowFunctions = exec::windowFunctions(); + ASSERT_TRUE(validateDefaultNamespacePrefix( + windowFunctions, prestoBuiltinFunctionPrefix)); +} } // namespace facebook::presto::test diff --git a/presto-native-execution/presto_cpp/main/types/FunctionMetadata.cpp b/presto-native-execution/presto_cpp/main/types/FunctionMetadata.cpp index a7749e1126dd2..e7aeccbcfff1a 100644 --- a/presto-native-execution/presto_cpp/main/types/FunctionMetadata.cpp +++ b/presto-native-execution/presto_cpp/main/types/FunctionMetadata.cpp @@ -86,7 +86,8 @@ const protocol::AggregationFunctionMetadata getAggregationFunctionMetadata( const std::string& name, const AggregateFunctionSignature& signature) { protocol::AggregationFunctionMetadata metadata; - metadata.intermediateType = signature.intermediateType().toString(); + metadata.intermediateType = + boost::algorithm::to_lower_copy(signature.intermediateType().toString()); metadata.isOrderSensitive = getAggregateFunctionEntry(name)->metadata.orderSensitive; return metadata; @@ -140,6 +141,24 @@ const protocol::RoutineCharacteristics getRoutineCharacteristics( return routineCharacteristics; } +const std::vector getTypeVariableConstraints( + const FunctionSignature& functionSignature) { + std::vector typeVariableConstraints; + const auto functionVariables = functionSignature.variables(); + for (const auto& [name, signature] : functionVariables) { + if (signature.isTypeParameter()) { + protocol::TypeVariableConstraint typeVariableConstraint; + typeVariableConstraint.name = + boost::algorithm::to_lower_copy(signature.name()); + typeVariableConstraint.orderableRequired = signature.orderableTypesOnly(); + typeVariableConstraint.comparableRequired = + signature.comparableTypesOnly(); + typeVariableConstraints.emplace_back(typeVariableConstraint); + } + } + return typeVariableConstraints; +} + std::optional buildFunctionMetadata( const std::string& name, const std::string& schema, @@ -152,7 +171,8 @@ std::optional buildFunctionMetadata( if (!isValidPrestoType(signature.returnType())) { return std::nullopt; } - metadata.outputType = signature.returnType().toString(); + metadata.outputType = + boost::algorithm::to_lower_copy(signature.returnType().toString()); const auto& argumentTypes = signature.argumentTypes(); std::vector paramTypes(argumentTypes.size()); @@ -160,11 +180,16 @@ std::optional buildFunctionMetadata( if (!isValidPrestoType(argumentTypes.at(i))) { return std::nullopt; } - paramTypes[i] = argumentTypes.at(i).toString(); + paramTypes[i] = + boost::algorithm::to_lower_copy(argumentTypes.at(i).toString()); } metadata.paramTypes = paramTypes; metadata.schema = schema; + metadata.variableArity = signature.variableArity(); metadata.routineCharacteristics = getRoutineCharacteristics(name, kind); + metadata.typeVariableConstraints = + std::make_shared>( + getTypeVariableConstraints(signature)); if (aggregateSignature) { metadata.aggregateMetadata = @@ -199,8 +224,22 @@ json buildAggregateMetadata( getWindowFunctionSignatures(name).has_value(), "Aggregate function {} not registered as a window function", name); + + // The functions returned by this endpoint are stored as SqlInvokedFunction + // objects, with SqlFunctionId serving as the primary key. SqlFunctionId is + // derived from both the functionName and argumentTypes parameters. Returning + // the same function twice—once as an aggregate function and once as a window + // function introduces ambiguity, as functionKind is not a component of + // SqlFunctionId. For any aggregate function utilized as a window function, + // the function’s metadata can be obtained from the associated aggregate + // function implementation for further processing. For additional information, + // refer to the following: • + // https://github.com/prestodb/presto/blob/master/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionId.java + // • + // https://github.com/prestodb/presto/blob/master/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedFunction.java + const std::vector kinds = { - protocol::FunctionKind::AGGREGATE, protocol::FunctionKind::WINDOW}; + protocol::FunctionKind::AGGREGATE}; json j = json::array(); json tj; for (const auto& kind : kinds) { diff --git a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.cpp b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.cpp index 2c2b2a3c5ea00..ff506dd936d27 100644 --- a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.cpp +++ b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.cpp @@ -14,6 +14,7 @@ #include "presto_cpp/main/types/PrestoToVeloxExpr.h" #include +#include "presto_cpp/main/common/Configs.h" #include "presto_cpp/presto_protocol/Base64Util.h" #include "velox/common/base/Exceptions.h" #include "velox/functions/prestosql/types/JsonType.h" @@ -33,24 +34,44 @@ std::string toJsonString(const T& value) { } std::string mapScalarFunction(const std::string& name) { + static const std::string prestoDefaultNamespacePrefix = + SystemConfig::instance()->prestoDefaultNamespacePrefix(); static const std::unordered_map kFunctionNames = { // Operator overrides: com.facebook.presto.common.function.OperatorType - {"presto.default.$operator$add", "presto.default.plus"}, - {"presto.default.$operator$between", "presto.default.between"}, - {"presto.default.$operator$divide", "presto.default.divide"}, - {"presto.default.$operator$equal", "presto.default.eq"}, - {"presto.default.$operator$greater_than", "presto.default.gt"}, - {"presto.default.$operator$greater_than_or_equal", "presto.default.gte"}, + {"presto.default.$operator$add", + util::addDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "plus")}, + {"presto.default.$operator$between", + util::addDefaultNamespacePrefix( + prestoDefaultNamespacePrefix, "between")}, + {"presto.default.$operator$divide", + util::addDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "divide")}, + {"presto.default.$operator$equal", + util::addDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "eq")}, + {"presto.default.$operator$greater_than", + util::addDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "gt")}, + {"presto.default.$operator$greater_than_or_equal", + util::addDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "gte")}, {"presto.default.$operator$is_distinct_from", - "presto.default.distinct_from"}, - {"presto.default.$operator$less_than", "presto.default.lt"}, - {"presto.default.$operator$less_than_or_equal", "presto.default.lte"}, - {"presto.default.$operator$modulus", "presto.default.mod"}, - {"presto.default.$operator$multiply", "presto.default.multiply"}, - {"presto.default.$operator$negation", "presto.default.negate"}, - {"presto.default.$operator$not_equal", "presto.default.neq"}, - {"presto.default.$operator$subtract", "presto.default.minus"}, - {"presto.default.$operator$subscript", "presto.default.subscript"}, + util::addDefaultNamespacePrefix( + prestoDefaultNamespacePrefix, "distinct_from")}, + {"presto.default.$operator$less_than", + util::addDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "lt")}, + {"presto.default.$operator$less_than_or_equal", + util::addDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "lte")}, + {"presto.default.$operator$modulus", + util::addDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "mod")}, + {"presto.default.$operator$multiply", + util::addDefaultNamespacePrefix( + prestoDefaultNamespacePrefix, "multiply")}, + {"presto.default.$operator$negation", + util::addDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "negate")}, + {"presto.default.$operator$not_equal", + util::addDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "neq")}, + {"presto.default.$operator$subtract", + util::addDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "minus")}, + {"presto.default.$operator$subscript", + util::addDefaultNamespacePrefix( + prestoDefaultNamespacePrefix, "subscript")}, // Special form function overrides. {"presto.default.in", "in"}, }; @@ -66,11 +87,15 @@ std::string mapScalarFunction(const std::string& name) { } std::string mapAggregateOrWindowFunction(const std::string& name) { + static const std::string prestoDefaultNamespacePrefix = + SystemConfig::instance()->prestoDefaultNamespacePrefix(); static const std::unordered_map kFunctionNames = { {"presto.default.$internal$max_data_size_for_stats", - "presto.default.max_data_size_for_stats"}, + util::addDefaultNamespacePrefix( + prestoDefaultNamespacePrefix, "max_data_size_for_stats")}, {"presto.default.$internal$sum_data_size_for_stats", - "presto.default.sum_data_size_for_stats"}, + util::addDefaultNamespacePrefix( + prestoDefaultNamespacePrefix, "sum_data_size_for_stats")}, }; std::string lowerCaseName = boost::to_lower_copy(name); auto it = kFunctionNames.find(name); @@ -167,6 +192,8 @@ std::optional convertCastToVarcharWithMaxLength( const std::string& returnType, const std::vector& args, bool nullOnFailure) { + static const std::string prestoDefaultNamespacePrefix = + SystemConfig::instance()->prestoDefaultNamespacePrefix(); if (nullOnFailure) { VELOX_NYI("TRY_CAST of varchar to {} is not supported.", returnType); } @@ -190,7 +217,7 @@ std::optional convertCastToVarcharWithMaxLength( std::make_shared(velox::BIGINT(), 1LL), std::make_shared(velox::BIGINT(), (int64_t)length), }, - "presto.default.substr"); + util::addDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "substr")); } /// Converts cast and try_cast functions to CastTypedExpr with nullOnFailure @@ -207,6 +234,8 @@ std::optional tryConvertCast( const std::string& returnType, const std::vector& args, const TypeParser* typeParser) { + static const std::string prestoDefaultNamespacePrefix = + SystemConfig::instance()->prestoDefaultNamespacePrefix(); static const char* kCast = "presto.default.$operator$cast"; static const char* kTryCast = "presto.default.try_cast"; static const char* kJsonToArrayCast = @@ -237,7 +266,10 @@ std::optional tryConvertCast( return std::make_shared( type, std::vector{std::make_shared( - velox::JSON(), args, "presto.default.json_parse")}, + velox::JSON(), + args, + util::addDefaultNamespacePrefix( + prestoDefaultNamespacePrefix, "json_parse"))}, false); } else { return std::nullopt; @@ -274,7 +306,6 @@ std::optional tryConvertTry( const std::vector& args, const TypeParser* typeParser) { static const char* kTry = "presto.default.$internal$try"; - if (signature.kind != protocol::FunctionKind::SCALAR) { return std::nullopt; } @@ -300,8 +331,11 @@ std::optional tryConvertLiteralArray( const std::vector& args, velox::memory::MemoryPool* pool, const TypeParser* typeParser) { + static const std::string prestoDefaultNamespacePrefix = + SystemConfig::instance()->prestoDefaultNamespacePrefix(); static const char* kLiteralArray = "presto.default.$literal$array"; - static const char* kFromBase64 = "presto.default.from_base64"; + static const std::string kFromBase64 = util::addDefaultNamespacePrefix( + prestoDefaultNamespacePrefix, "from_base64"); if (signature.kind != protocol::FunctionKind::SCALAR) { return std::nullopt; @@ -342,7 +376,10 @@ std::optional tryConvertLiteralArray( std::optional VeloxExprConverter::tryConvertDate( const protocol::CallExpression& pexpr) const { - static const char* kDate = "presto.default.date"; + static const std::string prestoDefaultNamespacePrefix = + SystemConfig::instance()->prestoDefaultNamespacePrefix(); + static const std::string kDate = + util::addDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "date"); auto builtin = std::static_pointer_cast( pexpr.functionHandle); @@ -363,8 +400,12 @@ std::optional VeloxExprConverter::tryConvertDate( std::optional VeloxExprConverter::tryConvertLike( const protocol::CallExpression& pexpr) const { - static const char* kLike = "presto.default.like"; - static const char* kLikePatternType = "presto.default.like_pattern"; + static const std::string prestoDefaultNamespacePrefix = + SystemConfig::instance()->prestoDefaultNamespacePrefix(); + static const std::string kLike = + util::addDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "like"); + static const std::string kLikePatternType = util::addDefaultNamespacePrefix( + prestoDefaultNamespacePrefix, "like_pattern"); static const char* kLikeReturnType = "LikePattern"; static const char* kCast = "presto.default.$operator$cast"; @@ -500,9 +541,13 @@ bool isTrueConstant(const TypedExprPtr& expression) { std::shared_ptr makeEqualsExpr( const TypedExprPtr& a, const TypedExprPtr& b) { + static const std::string prestoDefaultNamespacePrefix = + SystemConfig::instance()->prestoDefaultNamespacePrefix(); std::vector inputs{a, b}; return std::make_shared( - velox::BOOLEAN(), std::move(inputs), "presto.default.eq"); + velox::BOOLEAN(), + std::move(inputs), + util::addDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "eq")); } std::shared_ptr makeCastExpr( diff --git a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.h b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.h index f63a84ec35ad2..8526f6a8c9638 100644 --- a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.h +++ b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxExpr.h @@ -14,6 +14,8 @@ #pragma once #include +#include "presto_cpp/main/common/Configs.h" +#include "presto_cpp/main/common/Utils.h" #include "presto_cpp/main/types/TypeParser.h" #include "presto_cpp/presto_protocol/core/presto_protocol_core.h" #include "velox/core/Expressions.h" diff --git a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp index 31ea5f227eb8c..cdde0813df364 100644 --- a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp +++ b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp @@ -13,6 +13,7 @@ */ // clang-format off +#include "presto_cpp/main/common/Configs.h" #include "presto_cpp/main/types/PrestoToVeloxConnector.h" #include "presto_cpp/main/types/PrestoToVeloxQueryPlan.h" #include @@ -507,7 +508,10 @@ std::shared_ptr isFunctionCall( /// CallExpression. Returns nullptr if input expression is something else. std::shared_ptr isNot( const std::shared_ptr& expression) { - static const std::string_view kNot = "presto.default.not"; + static const std::string prestoDefaultNamespacePrefix = + SystemConfig::instance()->prestoDefaultNamespacePrefix(); + static const std::string kNot = + util::addDefaultNamespacePrefix(prestoDefaultNamespacePrefix, "not"); return isFunctionCall(expression, kNot); } @@ -1528,11 +1532,14 @@ namespace { core::WindowNode::Function makeRowNumberFunction( const protocol::VariableReferenceExpression& rowNumberVariable, const TypeParser& typeParser) { + static const std::string prestoDefaultNamespacePrefix = + SystemConfig::instance()->prestoDefaultNamespacePrefix(); core::WindowNode::Function function; function.functionCall = std::make_shared( stringToType(rowNumberVariable.type, typeParser), std::vector{}, - "presto.default.row_number"); + util::addDefaultNamespacePrefix( + prestoDefaultNamespacePrefix, "row_number")); function.frame.type = core::WindowNode::WindowType::kRows; function.frame.startType = core::WindowNode::BoundType::kUnboundedPreceding; diff --git a/presto-native-execution/presto_cpp/main/types/tests/FunctionMetadataTest.cpp b/presto-native-execution/presto_cpp/main/types/tests/FunctionMetadataTest.cpp index 433d6775afb4c..aaec19ac3634a 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/FunctionMetadataTest.cpp +++ b/presto-native-execution/presto_cpp/main/types/tests/FunctionMetadataTest.cpp @@ -61,7 +61,7 @@ class FunctionMetadataTest : public ::testing::Test { }; TEST_F(FunctionMetadataTest, approxMostFrequent) { - testFunction("approx_most_frequent", "ApproxMostFrequent.json", 12); + testFunction("approx_most_frequent", "ApproxMostFrequent.json", 6); } TEST_F(FunctionMetadataTest, arrayFrequency) { @@ -73,13 +73,17 @@ TEST_F(FunctionMetadataTest, combinations) { } TEST_F(FunctionMetadataTest, covarSamp) { - testFunction("covar_samp", "CovarSamp.json", 4); + testFunction("covar_samp", "CovarSamp.json", 2); } TEST_F(FunctionMetadataTest, elementAt) { testFunction("element_at", "ElementAt.json", 3); } +TEST_F(FunctionMetadataTest, greatest) { + testFunction("greatest", "Greatest.json", 13); +} + TEST_F(FunctionMetadataTest, lead) { testFunction("lead", "Lead.json", 3); } @@ -89,11 +93,11 @@ TEST_F(FunctionMetadataTest, ntile) { } TEST_F(FunctionMetadataTest, setAgg) { - testFunction("set_agg", "SetAgg.json", 2); + testFunction("set_agg", "SetAgg.json", 1); } TEST_F(FunctionMetadataTest, stddevSamp) { - testFunction("stddev_samp", "StddevSamp.json", 10); + testFunction("stddev_samp", "StddevSamp.json", 5); } TEST_F(FunctionMetadataTest, transformKeys) { @@ -101,5 +105,5 @@ TEST_F(FunctionMetadataTest, transformKeys) { } TEST_F(FunctionMetadataTest, variance) { - testFunction("variance", "Variance.json", 10); + testFunction("variance", "Variance.json", 5); } diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/ApproxMostFrequent.json b/presto-native-execution/presto_cpp/main/types/tests/data/ApproxMostFrequent.json index 4d1f793bda5a6..10e9a885cedfd 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/data/ApproxMostFrequent.json +++ b/presto-native-execution/presto_cpp/main/types/tests/data/ApproxMostFrequent.json @@ -18,7 +18,9 @@ "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "aggregateMetadata": { @@ -38,7 +40,9 @@ "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "aggregateMetadata": { @@ -58,7 +62,9 @@ "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "aggregateMetadata": { @@ -78,7 +84,9 @@ "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "aggregateMetadata": { @@ -98,7 +106,9 @@ "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "aggregateMetadata": { @@ -118,127 +128,9 @@ "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" - }, - { - "aggregateMetadata": { - "intermediateType": "row(bigint,bigint,array(boolean),array(bigint))", - "isOrderSensitive": true - }, - "docString": "presto.default.approx_most_frequent", - "functionKind": "WINDOW", - "outputType": "map(boolean,bigint)", - "paramTypes": [ - "bigint", - "boolean", - "bigint" - ], - "routineCharacteristics": { - "determinism": "DETERMINISTIC", - "language": "CPP", - "nullCallClause": "CALLED_ON_NULL_INPUT" - }, - "schema": "default" - }, - { - "aggregateMetadata": { - "intermediateType": "row(bigint,bigint,array(tinyint),array(bigint))", - "isOrderSensitive": true - }, - "docString": "presto.default.approx_most_frequent", - "functionKind": "WINDOW", - "outputType": "map(tinyint,bigint)", - "paramTypes": [ - "bigint", - "tinyint", - "bigint" - ], - "routineCharacteristics": { - "determinism": "DETERMINISTIC", - "language": "CPP", - "nullCallClause": "CALLED_ON_NULL_INPUT" - }, - "schema": "default" - }, - { - "aggregateMetadata": { - "intermediateType": "row(bigint,bigint,array(smallint),array(bigint))", - "isOrderSensitive": true - }, - "docString": "presto.default.approx_most_frequent", - "functionKind": "WINDOW", - "outputType": "map(smallint,bigint)", - "paramTypes": [ - "bigint", - "smallint", - "bigint" - ], - "routineCharacteristics": { - "determinism": "DETERMINISTIC", - "language": "CPP", - "nullCallClause": "CALLED_ON_NULL_INPUT" - }, - "schema": "default" - }, - { - "aggregateMetadata": { - "intermediateType": "row(bigint,bigint,array(integer),array(bigint))", - "isOrderSensitive": true - }, - "docString": "presto.default.approx_most_frequent", - "functionKind": "WINDOW", - "outputType": "map(integer,bigint)", - "paramTypes": [ - "bigint", - "integer", - "bigint" - ], - "routineCharacteristics": { - "determinism": "DETERMINISTIC", - "language": "CPP", - "nullCallClause": "CALLED_ON_NULL_INPUT" - }, - "schema": "default" - }, - { - "aggregateMetadata": { - "intermediateType": "row(bigint,bigint,array(bigint),array(bigint))", - "isOrderSensitive": true - }, - "docString": "presto.default.approx_most_frequent", - "functionKind": "WINDOW", - "outputType": "map(bigint,bigint)", - "paramTypes": [ - "bigint", - "bigint", - "bigint" - ], - "routineCharacteristics": { - "determinism": "DETERMINISTIC", - "language": "CPP", - "nullCallClause": "CALLED_ON_NULL_INPUT" - }, - "schema": "default" - }, - { - "aggregateMetadata": { - "intermediateType": "row(bigint,bigint,array(varchar),array(bigint))", - "isOrderSensitive": true - }, - "docString": "presto.default.approx_most_frequent", - "functionKind": "WINDOW", - "outputType": "map(varchar,bigint)", - "paramTypes": [ - "bigint", - "varchar", - "bigint" - ], - "routineCharacteristics": { - "determinism": "DETERMINISTIC", - "language": "CPP", - "nullCallClause": "CALLED_ON_NULL_INPUT" - }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false } ] } diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/ArrayFrequency.json b/presto-native-execution/presto_cpp/main/types/tests/data/ArrayFrequency.json index 485028cca4d6d..bf320fa8a3a6a 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/data/ArrayFrequency.json +++ b/presto-native-execution/presto_cpp/main/types/tests/data/ArrayFrequency.json @@ -12,7 +12,9 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "docString": "presto.default.array_frequency", @@ -26,7 +28,9 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "docString": "presto.default.array_frequency", @@ -40,7 +44,9 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "docString": "presto.default.array_frequency", @@ -54,7 +60,9 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "docString": "presto.default.array_frequency", @@ -68,7 +76,9 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "docString": "presto.default.array_frequency", @@ -82,7 +92,9 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "docString": "presto.default.array_frequency", @@ -96,7 +108,9 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "docString": "presto.default.array_frequency", @@ -110,7 +124,9 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "docString": "presto.default.array_frequency", @@ -124,7 +140,9 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "docString": "presto.default.array_frequency", @@ -138,7 +156,9 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false } ] } diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/Combinations.json b/presto-native-execution/presto_cpp/main/types/tests/data/Combinations.json index a05bc2243a364..7ff401c195ceb 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/data/Combinations.json +++ b/presto-native-execution/presto_cpp/main/types/tests/data/Combinations.json @@ -3,9 +3,9 @@ { "docString": "presto.default.combinations", "functionKind": "SCALAR", - "outputType": "array(array(__user_T1))", + "outputType": "array(array(__user_t1))", "paramTypes": [ - "array(__user_T1)", + "array(__user_t1)", "integer" ], "routineCharacteristics": { @@ -13,7 +13,17 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints": [ + { + "comparableRequired":false, + "name":"__user_t1", + "nonDecimalNumericRequired":false, + "orderableRequired":false, + "variadicBound":"" + } + ], + "variableArity":false }, { "docString": "presto.default.combinations", @@ -28,7 +38,9 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "docString": "presto.default.combinations", @@ -43,7 +55,9 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "docString": "presto.default.combinations", @@ -58,7 +72,9 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "docString": "presto.default.combinations", @@ -73,7 +89,9 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "docString": "presto.default.combinations", @@ -88,7 +106,9 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "docString": "presto.default.combinations", @@ -103,7 +123,9 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "docString": "presto.default.combinations", @@ -118,7 +140,9 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "docString": "presto.default.combinations", @@ -133,7 +157,9 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "docString": "presto.default.combinations", @@ -148,7 +174,9 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "docString": "presto.default.combinations", @@ -163,7 +191,9 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false } ] } diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/CovarSamp.json b/presto-native-execution/presto_cpp/main/types/tests/data/CovarSamp.json index 7b1d2d8e5adb0..ae39bef1e969f 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/data/CovarSamp.json +++ b/presto-native-execution/presto_cpp/main/types/tests/data/CovarSamp.json @@ -17,7 +17,9 @@ "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "aggregateMetadata": { @@ -36,45 +38,9 @@ "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" - }, - { - "aggregateMetadata": { - "intermediateType": "row(double,bigint,double,double)", - "isOrderSensitive": true - }, - "docString": "presto.default.covar_samp", - "functionKind": "WINDOW", - "outputType": "double", - "paramTypes": [ - "double", - "double" - ], - "routineCharacteristics": { - "determinism": "DETERMINISTIC", - "language": "CPP", - "nullCallClause": "CALLED_ON_NULL_INPUT" - }, - "schema": "default" - }, - { - "aggregateMetadata": { - "intermediateType": "row(double,bigint,double,double)", - "isOrderSensitive": true - }, - "docString": "presto.default.covar_samp", - "functionKind": "WINDOW", - "outputType": "real", - "paramTypes": [ - "real", - "real" - ], - "routineCharacteristics": { - "determinism": "DETERMINISTIC", - "language": "CPP", - "nullCallClause": "CALLED_ON_NULL_INPUT" - }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false } ] } diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/ElementAt.json b/presto-native-execution/presto_cpp/main/types/tests/data/ElementAt.json index ccd6405762abf..3956c7bfd1a98 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/data/ElementAt.json +++ b/presto-native-execution/presto_cpp/main/types/tests/data/ElementAt.json @@ -3,9 +3,9 @@ { "docString": "presto.default.element_at", "functionKind": "SCALAR", - "outputType": "T", + "outputType": "t", "paramTypes": [ - "array(T)", + "array(t)", "integer" ], "routineCharacteristics": { @@ -13,14 +13,24 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints": [ + { + "comparableRequired":false, + "name":"t", + "nonDecimalNumericRequired":false, + "orderableRequired":false, + "variadicBound":"" + } + ], + "variableArity":false }, { "docString": "presto.default.element_at", "functionKind": "SCALAR", - "outputType": "T", + "outputType": "t", "paramTypes": [ - "array(T)", + "array(t)", "bigint" ], "routineCharacteristics": { @@ -28,22 +38,49 @@ "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints": [ + { + "comparableRequired":false, + "name":"t", + "nonDecimalNumericRequired":false, + "orderableRequired":false, + "variadicBound":"" + } + ], + "variableArity":false }, { "docString": "presto.default.element_at", "functionKind": "SCALAR", - "outputType": "V", + "outputType": "v", "paramTypes": [ - "map(K,V)", - "K" + "map(k,v)", + "k" ], "routineCharacteristics": { "determinism": "DETERMINISTIC", "language": "CPP", "nullCallClause": "RETURNS_NULL_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints": [ + { + "comparableRequired":false, + "name":"v", + "nonDecimalNumericRequired":false, + "orderableRequired":false, + "variadicBound":"" + }, + { + "comparableRequired":false, + "name":"k", + "nonDecimalNumericRequired":false, + "orderableRequired":false, + "variadicBound":"" + } + ], + "variableArity":false } ] } diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/Greatest.json b/presto-native-execution/presto_cpp/main/types/tests/data/Greatest.json new file mode 100644 index 0000000000000..82070bade32d5 --- /dev/null +++ b/presto-native-execution/presto_cpp/main/types/tests/data/Greatest.json @@ -0,0 +1,185 @@ +{ + "greatest": [ + { + "docString":"presto.default.greatest", + "functionKind":"SCALAR", + "outputType":"timestamp with time zone", + "paramTypes":["timestamp with time zone","timestamp with time zone"], + "routineCharacteristics": { + "determinism":"DETERMINISTIC", + "language":"CPP", + "nullCallClause":"RETURNS_NULL_ON_NULL_INPUT" + }, + "schema":"default", + "typeVariableConstraints":[], + "variableArity":true}, + { + "docString":"presto.default.greatest", + "functionKind":"SCALAR", + "outputType":"double", + "paramTypes":["double","double"], + "routineCharacteristics": { + "determinism":"DETERMINISTIC", + "language":"CPP", + "nullCallClause":"RETURNS_NULL_ON_NULL_INPUT" + }, + "schema":"default", + "typeVariableConstraints":[], + "variableArity":true + }, + { + "docString":"presto.default.greatest", + "functionKind":"SCALAR", + "outputType":"varchar", + "paramTypes":["varchar","varchar"], + "routineCharacteristics": { + "determinism":"DETERMINISTIC", + "language":"CPP", + "nullCallClause":"RETURNS_NULL_ON_NULL_INPUT" + }, + "schema":"default", + "typeVariableConstraints":[], + "variableArity":true + }, + { + "docString":"presto.default.greatest", + "functionKind":"SCALAR", + "outputType":"real", + "paramTypes":["real","real"], + "routineCharacteristics": { + "determinism":"DETERMINISTIC", + "language":"CPP", + "nullCallClause":"RETURNS_NULL_ON_NULL_INPUT" + }, + "schema":"default", + "typeVariableConstraints":[], + "variableArity":true + }, + { + "docString":"presto.default.greatest", + "functionKind":"SCALAR", + "outputType":"timestamp", + "paramTypes":["timestamp","timestamp"], + "routineCharacteristics": { + "determinism":"DETERMINISTIC", + "language":"CPP", + "nullCallClause":"RETURNS_NULL_ON_NULL_INPUT" + }, + "schema":"default", + "typeVariableConstraints":[], + "variableArity":true + }, + { + "docString":"presto.default.greatest", + "functionKind":"SCALAR", + "outputType":"bigint", + "paramTypes":["bigint","bigint"], + "routineCharacteristics": { + "determinism":"DETERMINISTIC", + "language":"CPP", + "nullCallClause":"RETURNS_NULL_ON_NULL_INPUT" + }, + "schema":"default", + "typeVariableConstraints":[], + "variableArity":true + }, + { + "docString":"presto.default.greatest", + "functionKind":"SCALAR", + "outputType":"decimal(i1,i5)", + "paramTypes":["decimal(i1,i5)","decimal(i1,i5)"], + "routineCharacteristics": { + "determinism":"DETERMINISTIC", + "language":"CPP", + "nullCallClause":"RETURNS_NULL_ON_NULL_INPUT" + }, + "schema":"default", + "typeVariableConstraints":[], + "variableArity":true + }, + { + "docString":"presto.default.greatest", + "functionKind":"SCALAR", + "outputType":"boolean", + "paramTypes":["boolean","boolean"], + "routineCharacteristics": { + "determinism":"DETERMINISTIC", + "language":"CPP", + "nullCallClause":"RETURNS_NULL_ON_NULL_INPUT" + }, + "schema":"default", + "typeVariableConstraints":[], + "variableArity":true + }, + { + "docString":"presto.default.greatest", + "functionKind":"SCALAR", + "outputType":"smallint", + "paramTypes":["smallint","smallint"], + "routineCharacteristics": { + "determinism":"DETERMINISTIC", + "language":"CPP", + "nullCallClause":"RETURNS_NULL_ON_NULL_INPUT" + }, + "schema":"default", + "typeVariableConstraints":[], + "variableArity":true + }, + { + "docString":"presto.default.greatest", + "functionKind":"SCALAR", + "outputType":"date", + "paramTypes":["date","date"], + "routineCharacteristics": { + "determinism":"DETERMINISTIC", + "language":"CPP", + "nullCallClause":"RETURNS_NULL_ON_NULL_INPUT" + }, + "schema":"default", + "typeVariableConstraints":[], + "variableArity":true + }, + { + "docString":"presto.default.greatest", + "functionKind":"SCALAR", + "outputType":"tinyint", + "paramTypes":["tinyint","tinyint"], + "routineCharacteristics": { + "determinism":"DETERMINISTIC", + "language":"CPP", + "nullCallClause":"RETURNS_NULL_ON_NULL_INPUT" + }, + "schema":"default", + "typeVariableConstraints":[], + "variableArity":true + }, + { + "docString":"presto.default.greatest", + "functionKind":"SCALAR", + "outputType":"integer", + "paramTypes":["integer","integer"], + "routineCharacteristics": { + "determinism":"DETERMINISTIC", + "language":"CPP", + "nullCallClause":"RETURNS_NULL_ON_NULL_INPUT" + }, + "schema":"default", + "typeVariableConstraints":[], + "variableArity":true + }, + { + "docString":"presto.default.greatest", + "functionKind":"SCALAR", + "outputType":"ipaddress", + "paramTypes":["ipaddress","ipaddress"], + "routineCharacteristics": { + "determinism":"DETERMINISTIC", + "language":"CPP", + "nullCallClause":"RETURNS_NULL_ON_NULL_INPUT" + }, + "schema":"default", + "typeVariableConstraints":[], + "variableArity":true + } + ] +} \ No newline at end of file diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/Lead.json b/presto-native-execution/presto_cpp/main/types/tests/data/Lead.json index 6f06a1e8c6965..6af22cdc6dbc7 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/data/Lead.json +++ b/presto-native-execution/presto_cpp/main/types/tests/data/Lead.json @@ -3,23 +3,33 @@ { "docString": "presto.default.lead", "functionKind": "WINDOW", - "outputType": "T", + "outputType": "t", "paramTypes": [ - "T" + "t" ], "routineCharacteristics": { "determinism": "DETERMINISTIC", "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints": [ + { + "comparableRequired":false, + "name":"t", + "nonDecimalNumericRequired":false, + "orderableRequired":false, + "variadicBound":"" + } + ], + "variableArity":false }, { "docString": "presto.default.lead", "functionKind": "WINDOW", - "outputType": "T", + "outputType": "t", "paramTypes": [ - "T", + "t", "bigint" ], "routineCharacteristics": { @@ -27,23 +37,43 @@ "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints": [ + { + "comparableRequired":false, + "name":"t", + "nonDecimalNumericRequired":false, + "orderableRequired":false, + "variadicBound":"" + } + ], + "variableArity":false }, { "docString": "presto.default.lead", "functionKind": "WINDOW", - "outputType": "T", + "outputType": "t", "paramTypes": [ - "T", + "t", "bigint", - "T" + "t" ], "routineCharacteristics": { "determinism": "DETERMINISTIC", "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints": [ + { + "comparableRequired":false, + "name":"t", + "nonDecimalNumericRequired":false, + "orderableRequired":false, + "variadicBound":"" + } + ], + "variableArity":false } ] } diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/Ntile.json b/presto-native-execution/presto_cpp/main/types/tests/data/Ntile.json index 5685eccc25900..a8c38addc8772 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/data/Ntile.json +++ b/presto-native-execution/presto_cpp/main/types/tests/data/Ntile.json @@ -12,7 +12,9 @@ "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false } ] } diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/SetAgg.json b/presto-native-execution/presto_cpp/main/types/tests/data/SetAgg.json index 2fd34d4800426..bced64e34fab2 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/data/SetAgg.json +++ b/presto-native-execution/presto_cpp/main/types/tests/data/SetAgg.json @@ -2,39 +2,31 @@ "set_agg": [ { "aggregateMetadata": { - "intermediateType": "array(T)", + "intermediateType": "array(t)", "isOrderSensitive": true }, "docString": "presto.default.set_agg", "functionKind": "AGGREGATE", - "outputType": "array(T)", + "outputType": "array(t)", "paramTypes": [ - "T" + "t" ], "routineCharacteristics": { "determinism": "DETERMINISTIC", "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" - }, - { - "aggregateMetadata": { - "intermediateType": "array(T)", - "isOrderSensitive": true - }, - "docString": "presto.default.set_agg", - "functionKind": "WINDOW", - "outputType": "array(T)", - "paramTypes": [ - "T" + "schema": "default", + "typeVariableConstraints": [ + { + "comparableRequired":false, + "name":"t", + "nonDecimalNumericRequired":false, + "orderableRequired":false, + "variadicBound":"" + } ], - "routineCharacteristics": { - "determinism": "DETERMINISTIC", - "language": "CPP", - "nullCallClause": "CALLED_ON_NULL_INPUT" - }, - "schema": "default" + "variableArity":false } ] } diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/StddevSamp.json b/presto-native-execution/presto_cpp/main/types/tests/data/StddevSamp.json index 857d314a7d3a6..14103ae13a2f7 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/data/StddevSamp.json +++ b/presto-native-execution/presto_cpp/main/types/tests/data/StddevSamp.json @@ -16,7 +16,9 @@ "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "aggregateMetadata": { @@ -34,7 +36,9 @@ "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "aggregateMetadata": { @@ -52,7 +56,9 @@ "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "aggregateMetadata": { @@ -70,7 +76,9 @@ "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "aggregateMetadata": { @@ -88,97 +96,9 @@ "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" - }, - { - "aggregateMetadata": { - "intermediateType": "row(bigint,double,double)", - "isOrderSensitive": true - }, - "docString": "presto.default.stddev_samp", - "functionKind": "WINDOW", - "outputType": "double", - "paramTypes": [ - "smallint" - ], - "routineCharacteristics": { - "determinism": "DETERMINISTIC", - "language": "CPP", - "nullCallClause": "CALLED_ON_NULL_INPUT" - }, - "schema": "default" - }, - { - "aggregateMetadata": { - "intermediateType": "row(bigint,double,double)", - "isOrderSensitive": true - }, - "docString": "presto.default.stddev_samp", - "functionKind": "WINDOW", - "outputType": "double", - "paramTypes": [ - "integer" - ], - "routineCharacteristics": { - "determinism": "DETERMINISTIC", - "language": "CPP", - "nullCallClause": "CALLED_ON_NULL_INPUT" - }, - "schema": "default" - }, - { - "aggregateMetadata": { - "intermediateType": "row(bigint,double,double)", - "isOrderSensitive": true - }, - "docString": "presto.default.stddev_samp", - "functionKind": "WINDOW", - "outputType": "double", - "paramTypes": [ - "bigint" - ], - "routineCharacteristics": { - "determinism": "DETERMINISTIC", - "language": "CPP", - "nullCallClause": "CALLED_ON_NULL_INPUT" - }, - "schema": "default" - }, - { - "aggregateMetadata": { - "intermediateType": "row(bigint,double,double)", - "isOrderSensitive": true - }, - "docString": "presto.default.stddev_samp", - "functionKind": "WINDOW", - "outputType": "double", - "paramTypes": [ - "real" - ], - "routineCharacteristics": { - "determinism": "DETERMINISTIC", - "language": "CPP", - "nullCallClause": "CALLED_ON_NULL_INPUT" - }, - "schema": "default" - }, - { - "aggregateMetadata": { - "intermediateType": "row(bigint,double,double)", - "isOrderSensitive": true - }, - "docString": "presto.default.stddev_samp", - "functionKind": "WINDOW", - "outputType": "double", - "paramTypes": [ - "double" - ], - "routineCharacteristics": { - "determinism": "DETERMINISTIC", - "language": "CPP", - "nullCallClause": "CALLED_ON_NULL_INPUT" - }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false } ] } diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/TransformKeys.json b/presto-native-execution/presto_cpp/main/types/tests/data/TransformKeys.json index 2edcbdc98ee0e..16be6b35e5777 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/data/TransformKeys.json +++ b/presto-native-execution/presto_cpp/main/types/tests/data/TransformKeys.json @@ -3,17 +3,41 @@ { "docString": "presto.default.transform_keys", "functionKind": "SCALAR", - "outputType": "map(K2,V)", + "outputType": "map(k2,v)", "paramTypes": [ - "map(K1,V)", - "function(K1,V,K2)" + "map(k1,v)", + "function(k1,v,k2)" ], "routineCharacteristics": { "determinism": "DETERMINISTIC", "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints": [ + { + "comparableRequired":false, + "name":"v", + "nonDecimalNumericRequired":false, + "orderableRequired":false, + "variadicBound":"" + }, + { + "comparableRequired":false, + "name":"k2", + "nonDecimalNumericRequired":false, + "orderableRequired":false, + "variadicBound":"" + }, + { + "comparableRequired":false, + "name":"k1", + "nonDecimalNumericRequired":false, + "orderableRequired":false, + "variadicBound":"" + } + ], + "variableArity":false } ] } diff --git a/presto-native-execution/presto_cpp/main/types/tests/data/Variance.json b/presto-native-execution/presto_cpp/main/types/tests/data/Variance.json index ce491c58464bf..6e4549f845157 100644 --- a/presto-native-execution/presto_cpp/main/types/tests/data/Variance.json +++ b/presto-native-execution/presto_cpp/main/types/tests/data/Variance.json @@ -16,7 +16,9 @@ "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "aggregateMetadata": { @@ -34,7 +36,9 @@ "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "aggregateMetadata": { @@ -52,7 +56,9 @@ "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "aggregateMetadata": { @@ -70,7 +76,9 @@ "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "aggregateMetadata": { @@ -88,7 +96,9 @@ "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false }, { "aggregateMetadata": { @@ -106,79 +116,9 @@ "language": "CPP", "nullCallClause": "CALLED_ON_NULL_INPUT" }, - "schema": "default" - }, - { - "aggregateMetadata": { - "intermediateType": "row(bigint,double,double)", - "isOrderSensitive": true - }, - "docString": "presto.default.variance", - "functionKind": "WINDOW", - "outputType": "double", - "paramTypes": [ - "integer" - ], - "routineCharacteristics": { - "determinism": "DETERMINISTIC", - "language": "CPP", - "nullCallClause": "CALLED_ON_NULL_INPUT" - }, - "schema": "default" - }, - { - "aggregateMetadata": { - "intermediateType": "row(bigint,double,double)", - "isOrderSensitive": true - }, - "docString": "presto.default.variance", - "functionKind": "WINDOW", - "outputType": "double", - "paramTypes": [ - "bigint" - ], - "routineCharacteristics": { - "determinism": "DETERMINISTIC", - "language": "CPP", - "nullCallClause": "CALLED_ON_NULL_INPUT" - }, - "schema": "default" - }, - { - "aggregateMetadata": { - "intermediateType": "row(bigint,double,double)", - "isOrderSensitive": true - }, - "docString": "presto.default.variance", - "functionKind": "WINDOW", - "outputType": "double", - "paramTypes": [ - "real" - ], - "routineCharacteristics": { - "determinism": "DETERMINISTIC", - "language": "CPP", - "nullCallClause": "CALLED_ON_NULL_INPUT" - }, - "schema": "default" - }, - { - "aggregateMetadata": { - "intermediateType": "row(bigint,double,double)", - "isOrderSensitive": true - }, - "docString": "presto.default.variance", - "functionKind": "WINDOW", - "outputType": "double", - "paramTypes": [ - "double" - ], - "routineCharacteristics": { - "determinism": "DETERMINISTIC", - "language": "CPP", - "nullCallClause": "CALLED_ON_NULL_INPUT" - }, - "schema": "default" + "schema": "default", + "typeVariableConstraints":[], + "variableArity":false } ] } diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp index f74032ee76954..b0ecb4a22354a 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp @@ -108,6 +108,10 @@ void to_json(json& j, const std::shared_ptr& p) { j = *std::static_pointer_cast(p); return; } + if (type == "native") { + j = *std::static_pointer_cast(p); + return; + } if (type == "json_file") { j = *std::static_pointer_cast(p); return; @@ -131,6 +135,13 @@ void from_json(const json& j, std::shared_ptr& p) { p = std::static_pointer_cast(k); return; } + if (type == "native") { + std::shared_ptr k = + std::make_shared(); + j.get_to(*k); + p = std::static_pointer_cast(k); + return; + } if (type == "json_file") { std::shared_ptr k = std::make_shared(); @@ -1860,6 +1871,13 @@ void to_json(json& j, const SqlInvokedFunction& p) { "RoutineCharacteristics", "routineCharacteristics"); to_json_key(j, "body", p.body, "SqlInvokedFunction", "String", "body"); + to_json_key( + j, + "variableArity", + p.variableArity, + "SqlInvokedFunction", + "bool", + "variableArity"); to_json_key( j, "signature", @@ -1899,6 +1917,13 @@ void from_json(const json& j, SqlInvokedFunction& p) { "RoutineCharacteristics", "routineCharacteristics"); from_json_key(j, "body", p.body, "SqlInvokedFunction", "String", "body"); + from_json_key( + j, + "variableArity", + p.variableArity, + "SqlInvokedFunction", + "bool", + "variableArity"); from_json_key( j, "signature", @@ -5849,6 +5874,13 @@ void to_json(json& j, const JsonBasedUdfFunctionMetadata& p) { "JsonBasedUdfFunctionMetadata", "String", "schema"); + to_json_key( + j, + "variableArity", + p.variableArity, + "JsonBasedUdfFunctionMetadata", + "bool", + "variableArity"); to_json_key( j, "routineCharacteristics", @@ -5877,6 +5909,13 @@ void to_json(json& j, const JsonBasedUdfFunctionMetadata& p) { "JsonBasedUdfFunctionMetadata", "String", "version"); + to_json_key( + j, + "typeVariableConstraints", + p.typeVariableConstraints, + "JsonBasedUdfFunctionMetadata", + "List", + "typeVariableConstraints"); } void from_json(const json& j, JsonBasedUdfFunctionMetadata& p) { @@ -5915,6 +5954,13 @@ void from_json(const json& j, JsonBasedUdfFunctionMetadata& p) { "JsonBasedUdfFunctionMetadata", "String", "schema"); + from_json_key( + j, + "variableArity", + p.variableArity, + "JsonBasedUdfFunctionMetadata", + "bool", + "variableArity"); from_json_key( j, "routineCharacteristics", @@ -5943,6 +5989,13 @@ void from_json(const json& j, JsonBasedUdfFunctionMetadata& p) { "JsonBasedUdfFunctionMetadata", "String", "version"); + from_json_key( + j, + "typeVariableConstraints", + p.typeVariableConstraints, + "JsonBasedUdfFunctionMetadata", + "List", + "typeVariableConstraints"); } } // namespace facebook::presto::protocol /* diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h index 256e4dedf0f6a..907f160650cdb 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.h @@ -706,6 +706,7 @@ struct SqlInvokedFunction { String description = {}; RoutineCharacteristics routineCharacteristics = {}; String body = {}; + bool variableArity = {}; Signature signature = {}; SqlFunctionId functionId = {}; }; @@ -1507,10 +1508,12 @@ struct JsonBasedUdfFunctionMetadata { TypeSignature outputType = {}; List paramTypes = {}; String schema = {}; + bool variableArity = {}; RoutineCharacteristics routineCharacteristics = {}; std::shared_ptr aggregateMetadata = {}; std::shared_ptr functionId = {}; std::shared_ptr version = {}; + std::shared_ptr> typeVariableConstraints = {}; }; void to_json(json& j, const JsonBasedUdfFunctionMetadata& p); void from_json(const json& j, JsonBasedUdfFunctionMetadata& p); diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.yml b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.yml index 17b1f021207d5..32feb37fd501e 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.yml +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.yml @@ -174,6 +174,7 @@ AbstractClasses: super: JsonEncodedSubclass subclasses: - { name: BuiltInFunctionHandle, key: $static } + - { name: SqlFunctionHandle, key: native } - { name: SqlFunctionHandle, key: json_file } @@ -314,7 +315,7 @@ JavaClasses: - presto-main/src/main/java/com/facebook/presto/connector/system/SystemTableLayoutHandle.java - presto-main/src/main/java/com/facebook/presto/connector/system/SystemTransactionHandle.java - presto-spi/src/main/java/com/facebook/presto/spi/function/AggregationFunctionMetadata.java - - presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/JsonBasedUdfFunctionMetadata.java - presto-spi/src/main/java/com/facebook/presto/spi/plan/ExchangeEncoding.java - presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/nativechecker/PlanConversionFailureInfo.java - presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/nativechecker/PlanConversionResponse.java + - presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/JsonBasedUdfFunctionMetadata.java diff --git a/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.yml b/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.yml index b4f4308f550c1..2cda40445c76e 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.yml +++ b/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.yml @@ -174,6 +174,7 @@ AbstractClasses: subclasses: - { name: BuiltInFunctionHandle, key: $static } - { name: SqlFunctionHandle, key: json_file } + - { name: SqlFunctionHandle, key: native } JavaClasses: - presto-spi/src/main/java/com/facebook/presto/spi/ErrorCause.java @@ -364,4 +365,4 @@ JavaClasses: - presto-main/src/main/java/com/facebook/presto/connector/system/SystemTableLayoutHandle.java - presto-main/src/main/java/com/facebook/presto/connector/system/SystemTransactionHandle.java - presto-spi/src/main/java/com/facebook/presto/spi/function/AggregationFunctionMetadata.java - - presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/JsonBasedUdfFunctionMetadata.java + - presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/JsonBasedUdfFunctionMetadata.java diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/NativeQueryRunnerUtils.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/NativeQueryRunnerUtils.java index 3afa67b275c22..13443d5219198 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/NativeQueryRunnerUtils.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/NativeQueryRunnerUtils.java @@ -59,6 +59,7 @@ public static Map getNativeSidecarProperties() { return ImmutableMap.builder() .put("coordinator-sidecar-enabled", "true") + .put("presto.default-namespace", "native.default") .build(); } @@ -431,7 +432,8 @@ public static void createTableToTestHiddenColumns(QueryRunner queryRunner) try { TimeUnit.SECONDS.sleep(2); } - catch (InterruptedException e) { } + catch (InterruptedException e) { + } queryRunner.execute("INSERT INTO test_hidden_columns SELECT * FROM region where regionkey = 1"); } } diff --git a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java index 58431a64f7ca7..825fbb1bc4387 100644 --- a/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java +++ b/presto-native-execution/src/test/java/com/facebook/presto/nativeworker/PrestoNativeQueryRunnerUtils.java @@ -513,7 +513,8 @@ public static Optional> getExternalWorkerLaunc if (isCoordinatorSidecarEnabled) { configProperties = format("%s%n" + - "native-sidecar=true%n", configProperties); + "native-sidecar=true%n" + + "presto.default-namespace=native.default%n", configProperties); } if (remoteFunctionServerUds.isPresent()) { diff --git a/presto-native-sidecar-plugin/pom.xml b/presto-native-sidecar-plugin/pom.xml index b29409050be23..6b8f1badd289d 100644 --- a/presto-native-sidecar-plugin/pom.xml +++ b/presto-native-sidecar-plugin/pom.xml @@ -14,6 +14,8 @@ ${project.parent.basedir} + 1.8 + 1.8 @@ -42,6 +44,11 @@ guice + + javax.inject + javax.inject + + com.facebook.airlift configuration @@ -57,6 +64,12 @@ okhttp + + com.facebook.presto + presto-function-namespace-managers-common + ${project.version} + + com.facebook.presto diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/NativeSidecarPlugin.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/NativeSidecarPlugin.java index c719f5a2db809..7da7ba4d4fbb9 100644 --- a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/NativeSidecarPlugin.java +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/NativeSidecarPlugin.java @@ -13,9 +13,11 @@ */ package com.facebook.presto.sidecar; +import com.facebook.presto.sidecar.functionNamespace.NativeFunctionNamespaceManagerFactory; import com.facebook.presto.sidecar.nativechecker.NativePlanCheckerProviderFactory; import com.facebook.presto.sidecar.sessionpropertyproviders.NativeSystemSessionPropertyProviderFactory; import com.facebook.presto.spi.CoordinatorPlugin; +import com.facebook.presto.spi.function.FunctionNamespaceManagerFactory; import com.facebook.presto.spi.plan.PlanCheckerProviderFactory; import com.facebook.presto.spi.session.WorkerSessionPropertyProviderFactory; import com.google.common.collect.ImmutableList; @@ -35,6 +37,12 @@ public Iterable getPlanCheckerProviderFactories() return ImmutableList.of(new NativePlanCheckerProviderFactory(getClassLoader())); } + @Override + public Iterable getFunctionNamespaceManagerFactories() + { + return ImmutableList.of(new NativeFunctionNamespaceManagerFactory()); + } + private static ClassLoader getClassLoader() { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/FunctionDefinitionProvider.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/FunctionDefinitionProvider.java new file mode 100644 index 0000000000000..0badf407e1a7c --- /dev/null +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/FunctionDefinitionProvider.java @@ -0,0 +1,23 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.presto.sidecar.functionNamespace; + +import com.facebook.presto.functionNamespace.UdfFunctionSignatureMap; +import com.facebook.presto.spi.NodeManager; + +public interface FunctionDefinitionProvider +{ + UdfFunctionSignatureMap getUdfDefinition(NodeManager nodeManager); +} diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionDefinitionProvider.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionDefinitionProvider.java new file mode 100644 index 0000000000000..310cd9ebda2e6 --- /dev/null +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionDefinitionProvider.java @@ -0,0 +1,83 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sidecar.functionNamespace; + +import com.facebook.airlift.http.client.HttpClient; +import com.facebook.airlift.http.client.HttpUriBuilder; +import com.facebook.airlift.http.client.Request; +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.log.Logger; +import com.facebook.presto.functionNamespace.JsonBasedUdfFunctionMetadata; +import com.facebook.presto.functionNamespace.UdfFunctionSignatureMap; +import com.facebook.presto.sidecar.ForSidecarInfo; +import com.facebook.presto.spi.Node; +import com.facebook.presto.spi.NodeManager; +import com.facebook.presto.spi.PrestoException; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; + +import java.net.URI; +import java.util.List; +import java.util.Map; + +import static com.facebook.airlift.http.client.JsonResponseHandler.createJsonResponseHandler; +import static com.facebook.airlift.http.client.Request.Builder.prepareGet; +import static com.facebook.presto.spi.StandardErrorCode.INVALID_ARGUMENTS; +import static java.util.Objects.requireNonNull; + +public class NativeFunctionDefinitionProvider + implements FunctionDefinitionProvider +{ + private static final Logger log = Logger.get(NativeFunctionDefinitionProvider.class); + private final JsonCodec>> nativeFunctionSignatureMapJsonCodec; + private final NodeManager nodeManager; + private final HttpClient httpClient; + private static final String FUNCTION_SIGNATURES_ENDPOINT = "/v1/functions"; + + @Inject + public NativeFunctionDefinitionProvider( + @ForSidecarInfo HttpClient httpClient, + JsonCodec>> nativeFunctionSignatureMapJsonCodec, + NodeManager nodeManager) + { + this.nativeFunctionSignatureMapJsonCodec = + requireNonNull(nativeFunctionSignatureMapJsonCodec, "nativeFunctionSignatureMapJsonCodec is null"); + this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); + this.httpClient = requireNonNull(httpClient, "typeManager is null"); + } + + @Override + public UdfFunctionSignatureMap getUdfDefinition(NodeManager nodeManager) + { + try { + Request request = prepareGet().setUri(getSidecarLocation()).build(); + Map> nativeFunctionSignatureMap = httpClient.execute(request, createJsonResponseHandler(nativeFunctionSignatureMapJsonCodec)); + return new UdfFunctionSignatureMap(ImmutableMap.copyOf(nativeFunctionSignatureMap)); + } + catch (Exception e) { + throw new PrestoException(INVALID_ARGUMENTS, "Failed to get functions from sidecar.", e); + } + } + + private URI getSidecarLocation() + { + Node sidecarNode = nodeManager.getSidecarNode(); + return HttpUriBuilder.uriBuilder() + .scheme("http") + .host(sidecarNode.getHost()) + .port(sidecarNode.getHostAndPort().getPort()) + .appendPath(FUNCTION_SIGNATURES_ENDPOINT) + .build(); + } +} diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionHandle.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionHandle.java new file mode 100644 index 0000000000000..ae17d3f0c2995 --- /dev/null +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionHandle.java @@ -0,0 +1,103 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sidecar.functionNamespace; + +import com.facebook.presto.common.CatalogSchemaName; +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionHandleResolver; +import com.facebook.presto.spi.function.FunctionKind; +import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunctionHandle; +import com.facebook.presto.spi.function.SqlFunctionId; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Objects; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class NativeFunctionHandle + extends SqlFunctionHandle +{ + private final Signature signature; + + @JsonCreator + public NativeFunctionHandle(@JsonProperty("signature") Signature signature) + { + // Todo: hardcoding version as "1" + super(new SqlFunctionId(signature.getName(), signature.getArgumentTypes()), "1"); + this.signature = requireNonNull(signature, "signature is null"); + checkArgument(signature.getTypeVariableConstraints().isEmpty(), "%s has unbound type parameters", signature); + } + + @JsonProperty + public Signature getSignature() + { + return signature; + } + + @Override + public String getName() + { + return signature.getName().toString(); + } + + @Override + public FunctionKind getKind() + { + return signature.getKind(); + } + + @Override + public CatalogSchemaName getCatalogSchemaName() + { + return signature.getName().getCatalogSchemaName(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + NativeFunctionHandle that = (NativeFunctionHandle) o; + return Objects.equals(signature, that.signature); + } + + @Override + public int hashCode() + { + return Objects.hash(signature); + } + + @Override + public String toString() + { + return signature.toString(); + } + + public static class Resolver + implements FunctionHandleResolver + { + @Override + public Class getFunctionHandleClass() + { + return NativeFunctionHandle.class; + } + } +} diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManager.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManager.java new file mode 100644 index 0000000000000..024f21d0ec3af --- /dev/null +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManager.java @@ -0,0 +1,471 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sidecar.functionNamespace; + +import com.facebook.airlift.log.Logger; +import com.facebook.presto.common.CatalogSchemaName; +import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.common.type.NamedTypeSignature; +import com.facebook.presto.common.type.StandardTypes; +import com.facebook.presto.common.type.Type; +import com.facebook.presto.common.type.TypeManager; +import com.facebook.presto.common.type.TypeSignature; +import com.facebook.presto.common.type.TypeSignatureParameter; +import com.facebook.presto.common.type.UserDefinedType; +import com.facebook.presto.functionNamespace.AbstractSqlInvokedFunctionNamespaceManager; +import com.facebook.presto.functionNamespace.JsonBasedUdfFunctionMetadata; +import com.facebook.presto.functionNamespace.ServingCatalog; +import com.facebook.presto.functionNamespace.SqlInvokedFunctionNamespaceManagerConfig; +import com.facebook.presto.functionNamespace.UdfFunctionSignatureMap; +import com.facebook.presto.functionNamespace.execution.SqlFunctionExecutors; +import com.facebook.presto.spi.NodeManager; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.function.AggregationFunctionImplementation; +import com.facebook.presto.spi.function.AggregationFunctionMetadata; +import com.facebook.presto.spi.function.AlterRoutineCharacteristics; +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.FunctionNamespaceTransactionHandle; +import com.facebook.presto.spi.function.Parameter; +import com.facebook.presto.spi.function.ScalarFunctionImplementation; +import com.facebook.presto.spi.function.Signature; +import com.facebook.presto.spi.function.SqlFunction; +import com.facebook.presto.spi.function.SqlFunctionHandle; +import com.facebook.presto.spi.function.SqlFunctionId; +import com.facebook.presto.spi.function.SqlFunctionSupplier; +import com.facebook.presto.spi.function.SqlInvokedAggregationFunctionImplementation; +import com.facebook.presto.spi.function.SqlInvokedFunction; +import com.facebook.presto.spi.function.TypeVariableConstraint; +import com.google.common.base.Suppliers; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.UncheckedExecutionException; + +import javax.inject.Inject; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +import static com.facebook.presto.common.type.TypeSignatureUtils.resolveIntermediateType; +import static com.facebook.presto.spi.StandardErrorCode.DUPLICATE_FUNCTION_ERROR; +import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static com.facebook.presto.spi.StandardErrorCode.GENERIC_USER_ERROR; +import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; +import static com.facebook.presto.spi.function.FunctionVersion.notVersioned; +import static com.facebook.presto.spi.function.RoutineCharacteristics.Language.CPP; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.MoreCollectors.onlyElement; +import static java.lang.Long.parseLong; +import static java.lang.String.format; +import static java.util.Collections.unmodifiableMap; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.HOURS; + +public class NativeFunctionNamespaceManager + extends AbstractSqlInvokedFunctionNamespaceManager +{ + private static final Logger log = Logger.get(NativeFunctionNamespaceManager.class); + private final Map userDefinedTypes = new ConcurrentHashMap<>(); + private final Map aggregationImplementationByHandle = new ConcurrentHashMap<>(); + private final FunctionDefinitionProvider functionDefinitionProvider; + private final NodeManager nodeManager; + private final Map functions = new ConcurrentHashMap<>(); + private final Supplier> memoizedFunctionsSupplier; + private final FunctionMetadataManager functionMetadataManager; + private final LoadingCache specializedFunctionKeyCache; + + @Inject + public NativeFunctionNamespaceManager( + @ServingCatalog String catalogName, + SqlFunctionExecutors sqlFunctionExecutors, + SqlInvokedFunctionNamespaceManagerConfig config, + FunctionDefinitionProvider functionDefinitionProvider, + NodeManager nodeManager, + FunctionMetadataManager functionMetadataManager) + { + super(catalogName, sqlFunctionExecutors, config); + this.functionDefinitionProvider = requireNonNull(functionDefinitionProvider, "functionDefinitionProvider is null"); + this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); + this.memoizedFunctionsSupplier = Suppliers.memoizeWithExpiration(this::bootstrapNamespace, + config.getFunctionCacheExpiration().toMillis(), TimeUnit.MILLISECONDS); + this.functionMetadataManager = requireNonNull(functionMetadataManager, "functionMetadataManager is null"); + this.specializedFunctionKeyCache = CacheBuilder.newBuilder() + .maximumSize(1000) + .expireAfterWrite(1, HOURS) + .build(CacheLoader.from(this::doGetSpecializedFunctionKey)); + } + + private SqlFunctionSupplier doGetSpecializedFunctionKey(Signature signature) + { + return functionMetadataManager.getSpecializedFunctionKey(signature); + } + + private synchronized Map bootstrapNamespace() + { + functions.clear(); + UdfFunctionSignatureMap nativeFunctionSignatureMap = functionDefinitionProvider.getUdfDefinition(nodeManager); + if (nativeFunctionSignatureMap == null || nativeFunctionSignatureMap.isEmpty()) { + return ImmutableMap.of(); + } + populateNamespaceManager(nativeFunctionSignatureMap); + checkArgument(!functions.isEmpty(), "functions map is empty !"); + return unmodifiableMap(functions); + } + + private synchronized void populateNamespaceManager(UdfFunctionSignatureMap udfFunctionSignatureMap) + { + Map> udfSignatureMap = udfFunctionSignatureMap.getUDFSignatureMap(); + udfSignatureMap.forEach((name, metaInfoList) -> { + List functions = metaInfoList.stream().map(metaInfo -> createSqlInvokedFunction(name, metaInfo)).collect(toImmutableList()); + functions.forEach(function -> createFunction(function, false)); + }); + } + + @Override + public final AggregationFunctionImplementation getAggregateFunctionImplementation(FunctionHandle functionHandle, TypeManager typeManager) + { + checkCatalog(functionHandle); + checkArgument(functionHandle instanceof SqlFunctionHandle, "Unsupported FunctionHandle type '%s'", functionHandle.getClass().getSimpleName()); + + SqlFunctionHandle sqlFunctionHandle = (SqlFunctionHandle) functionHandle; + if (aggregationImplementationByHandle.containsKey(sqlFunctionHandle)) { + return aggregationImplementationByHandle.get(sqlFunctionHandle); + } + if (functionHandle instanceof NativeFunctionHandle) { + return processNativeFunctionHandle((NativeFunctionHandle) sqlFunctionHandle, typeManager); + } + else { + return processSqlFunctionHandle(sqlFunctionHandle, typeManager); + } + } + + private AggregationFunctionImplementation processNativeFunctionHandle(NativeFunctionHandle nativeFunctionHandle, TypeManager typeManager) + { + Signature signature = nativeFunctionHandle.getSignature(); + SqlFunction function = getSqlFunctionFromSignature(signature); + SqlInvokedFunction sqlFunction = (SqlInvokedFunction) function; + + checkArgument( + sqlFunction.getAggregationMetadata().isPresent(), + "Need aggregationMetadata to get aggregation function implementation"); + + AggregationFunctionMetadata aggregationMetadata = sqlFunction.getAggregationMetadata().get(); + TypeSignature intermediateType = aggregationMetadata.getIntermediateType(); + TypeSignature resolvedIntermediateType = resolveIntermediateType( + intermediateType, sqlFunction.getFunctionId().getArgumentTypes(), signature.getArgumentTypes()); + List parameters = signature.getArgumentTypes().stream().map( + (typeManager::getType)).collect(toImmutableList()); + aggregationImplementationByHandle.put( + nativeFunctionHandle, + new SqlInvokedAggregationFunctionImplementation( + typeManager.getType(resolvedIntermediateType), + typeManager.getType(signature.getReturnType()), + aggregationMetadata.isOrderSensitive(), + parameters)); + return aggregationImplementationByHandle.get(nativeFunctionHandle); + } + + private AggregationFunctionImplementation processSqlFunctionHandle(SqlFunctionHandle sqlFunctionHandle, TypeManager typeManager) + { + SqlFunctionId functionId = sqlFunctionHandle.getFunctionId(); + if (!memoizedFunctionsSupplier.get().containsKey(functionId)) { + throw new PrestoException(GENERIC_USER_ERROR, format("Function '%s' is missing from cache", functionId.getId())); + } + + aggregationImplementationByHandle.put( + sqlFunctionHandle, + sqlInvokedFunctionToAggregationImplementation(memoizedFunctionsSupplier.get().get(functionId), typeManager)); + return aggregationImplementationByHandle.get(sqlFunctionHandle); + } + + protected synchronized SqlInvokedFunction createSqlInvokedFunction(String functionName, JsonBasedUdfFunctionMetadata jsonBasedUdfFunctionMetaData) + { + checkState(jsonBasedUdfFunctionMetaData.getRoutineCharacteristics().getLanguage().equals(CPP), "NativeFunctionNamespaceManager only supports CPP UDF"); + QualifiedObjectName qualifiedFunctionName = QualifiedObjectName.valueOf(new CatalogSchemaName(getCatalogName(), jsonBasedUdfFunctionMetaData.getSchema()), functionName); + List parameterNameList = jsonBasedUdfFunctionMetaData.getParamNames(); + List parameterTypeList = convertApplicableTypeToVariable(jsonBasedUdfFunctionMetaData.getParamTypes()); + List typeVariableConstraintsList = jsonBasedUdfFunctionMetaData.getTypeVariableConstraints().isPresent() ? + jsonBasedUdfFunctionMetaData.getTypeVariableConstraints().get() : Collections.emptyList(); + + TypeSignature outputType = convertApplicableTypeToVariable(jsonBasedUdfFunctionMetaData.getOutputType()); + ImmutableList.Builder parameterBuilder = ImmutableList.builder(); + for (int i = 0; i < parameterNameList.size(); i++) { + parameterBuilder.add(new Parameter(parameterNameList.get(i), parameterTypeList.get(i))); + } + + Optional aggregationFunctionMetadata = + jsonBasedUdfFunctionMetaData.getAggregateMetadata() + .map(metadata -> new AggregationFunctionMetadata( + convertApplicableTypeToVariable(metadata.getIntermediateType()), + metadata.isOrderSensitive())); + + return new SqlInvokedFunction( + qualifiedFunctionName, + parameterBuilder.build(), + typeVariableConstraintsList, + outputType, + jsonBasedUdfFunctionMetaData.getDocString(), + jsonBasedUdfFunctionMetaData.getRoutineCharacteristics(), + "", + jsonBasedUdfFunctionMetaData.getVariableArity(), + notVersioned(), + jsonBasedUdfFunctionMetaData.getFunctionKind(), + aggregationFunctionMetadata); + } + + @Override + protected Collection fetchFunctionsDirect(QualifiedObjectName functionName) + { + return memoizedFunctionsSupplier.get().values().stream() + .filter(function -> function.getSignature().getName().equals(functionName)) + .collect(toImmutableList()); + } + + @Override + protected UserDefinedType fetchUserDefinedTypeDirect(QualifiedObjectName typeName) + { + return userDefinedTypes.get(typeName); + } + + @Override + protected FunctionMetadata fetchFunctionMetadataDirect(SqlFunctionHandle functionHandle) + { + if (functionHandle instanceof NativeFunctionHandle) { + return getMetadataFromNativeFunctionHandle(functionHandle); + } + + return fetchFunctionsDirect(functionHandle.getFunctionId().getFunctionName()).stream() + .filter(function -> function.getRequiredFunctionHandle().equals(functionHandle)) + .map(this::sqlInvokedFunctionToMetadata).collect(onlyElement()); + } + + @Override + protected ScalarFunctionImplementation fetchFunctionImplementationDirect(SqlFunctionHandle functionHandle) + { + return fetchFunctionsDirect(functionHandle.getFunctionId().getFunctionName()).stream() + .filter(function -> function.getRequiredFunctionHandle().equals(functionHandle)) + .map(this::sqlInvokedFunctionToImplementation) + .collect(onlyElement()); + } + + @Override + public synchronized void createFunction(SqlInvokedFunction function, boolean replace) + { + checkFunctionLanguageSupported(function); + SqlFunctionId functionId = function.getFunctionId(); + if (!replace && functions.containsKey(function.getFunctionId())) { + throw new PrestoException(DUPLICATE_FUNCTION_ERROR, format("Function '%s' already exists", functionId.getId())); + } + + SqlInvokedFunction replacedFunction = functions.get(functionId); + long version = 1; + if (replacedFunction != null) { + version = parseLong(replacedFunction.getRequiredVersion()) + 1; + } + functions.put(functionId, function.withVersion(String.valueOf(version))); + } + + @Override + public void alterFunction(QualifiedObjectName functionName, Optional> parameterTypes, AlterRoutineCharacteristics alterRoutineCharacteristics) + { + throw new PrestoException(NOT_SUPPORTED, "Alter Function is not supported in NativeFunctionNamespaceManager"); + } + + @Override + public void dropFunction(QualifiedObjectName functionName, Optional> parameterTypes, boolean exists) + { + throw new PrestoException(NOT_SUPPORTED, "Drop Function is not supported in NativeFunctionNamespaceManager"); + } + + @Override + public Collection listFunctions(Optional likePattern, Optional escape) + { + return memoizedFunctionsSupplier.get().values(); + } + + @Override + public void addUserDefinedType(UserDefinedType userDefinedType) + { + QualifiedObjectName name = userDefinedType.getUserDefinedTypeName(); + checkArgument( + !userDefinedTypes.containsKey(name), + "Parametric type %s already registered", + name); + userDefinedTypes.put(name, userDefinedType); + } + + @Override + public final FunctionHandle getFunctionHandle(Optional transactionHandle, Signature signature) + { + FunctionHandle functionHandle = super.getFunctionHandle(transactionHandle, signature); + + // only handle generic variadic signatures here , for normal signature we use the AbstractSqlInvokedFunctionNamespaceManager function handle. + if (functionHandle == null) { + return new NativeFunctionHandle(signature); + } + return functionHandle; + } + + // Todo: Improve the handling of parameter type differentiation in native execution. + // HACK: Currently, we lack support for correctly identifying the parameterKind, specifically between TYPE and VARIABLE, + // in native execution. The following utility functions help bridge this gap by parsing the type signature and verifying whether its base + // and parameters are of a supported type. The valid types list are non - parametric types that Presto supports. + + public static TypeSignature convertApplicableTypeToVariable(TypeSignature typeSignature) + { + List typeSignaturesList = convertApplicableTypeToVariable(ImmutableList.of(typeSignature)); + checkArgument(!typeSignaturesList.isEmpty(), "Type signature list is empty for : " + typeSignature); + return typeSignaturesList.get(0); + } + + public static List convertApplicableTypeToVariable(List typeSignatures) + { + List newTypeSignaturesList = new ArrayList<>(); + for (TypeSignature typeSignature : typeSignatures) { + if (!typeSignature.getParameters().isEmpty()) { + TypeSignature newTypeSignature = + new TypeSignature( + typeSignature.getBase(), + getTypeSignatureParameters( + typeSignature, + typeSignature.getParameters())); + newTypeSignaturesList.add(newTypeSignature); + } + else { + newTypeSignaturesList.add(typeSignature); + } + } + return newTypeSignaturesList; + } + + private static List getTypeSignatureParameters( + TypeSignature typeSignature, + List typeSignatureParameterList) + { + List newParameterTypeList = new ArrayList<>(); + for (TypeSignatureParameter parameter : typeSignatureParameterList) { + if (parameter.isLongLiteral()) { + newParameterTypeList.add(parameter); + continue; + } + + boolean isNamedTypeSignature = parameter.isNamedTypeSignature(); + TypeSignature parameterTypeSignature; + // If it's a named type signatures only in the case of row signature types. + if (isNamedTypeSignature) { + parameterTypeSignature = parameter.getNamedTypeSignature().getTypeSignature(); + } + else { + parameterTypeSignature = parameter.getTypeSignature(); + } + + if (parameterTypeSignature.getParameters().isEmpty()) { + boolean changeTypeToVariable = isDecimalTypeBase(typeSignature.getBase()); + if (changeTypeToVariable) { + newParameterTypeList.add( + TypeSignatureParameter.of(parameterTypeSignature.getBase())); + } + else { + if (isNamedTypeSignature) { + newParameterTypeList.add(TypeSignatureParameter.of(parameter.getNamedTypeSignature())); + } + else { + newParameterTypeList.add(TypeSignatureParameter.of(parameterTypeSignature)); + } + } + } + else { + TypeSignature newTypeSignature = + new TypeSignature( + parameterTypeSignature.getBase(), + getTypeSignatureParameters( + parameterTypeSignature.getStandardTypeSignature(), + parameterTypeSignature.getParameters())); + if (isNamedTypeSignature) { + newParameterTypeList.add( + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.empty(), + newTypeSignature))); + } + else { + newParameterTypeList.add(TypeSignatureParameter.of(newTypeSignature)); + } + } + } + return newParameterTypeList; + } + + private static boolean isDecimalTypeBase(String typeBase) + { + return typeBase.equals(StandardTypes.DECIMAL); + } + + // Hack ends here + + private SqlFunction getSqlFunctionFromSignature(Signature signature) + { + try { + return specializedFunctionKeyCache.getUnchecked(signature).getFunction(); + } + catch (UncheckedExecutionException e) { + throw convertToPrestoException(e, format("Error getting FunctionMetadata for signature: %s", signature)); + } + } + + private FunctionMetadata getMetadataFromNativeFunctionHandle(SqlFunctionHandle functionHandle) + { + NativeFunctionHandle nativeFunctionHandle = (NativeFunctionHandle) functionHandle; + Signature signature = nativeFunctionHandle.getSignature(); + SqlFunction function = getSqlFunctionFromSignature(signature); + + SqlInvokedFunction sqlFunction = (SqlInvokedFunction) function; + return new FunctionMetadata( + signature.getName(), + signature.getArgumentTypes(), + sqlFunction.getParameters().stream() + .map(Parameter::getName) + .collect(toImmutableList()), + signature.getReturnType(), + function.getSignature().getKind(), + sqlFunction.getRoutineCharacteristics().getLanguage(), + getFunctionImplementationType(sqlFunction), + function.isDeterministic(), + function.isCalledOnNullInput(), + sqlFunction.getVersion(), + function.getComplexTypeFunctionDescriptor()); + } + + private static PrestoException convertToPrestoException(UncheckedExecutionException exception, String failureMessage) + { + Throwable cause = exception.getCause(); + if (cause instanceof PrestoException) { + return (PrestoException) cause; + } + return new PrestoException(GENERIC_INTERNAL_ERROR, failureMessage, cause); + } +} diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManagerFactory.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManagerFactory.java new file mode 100644 index 0000000000000..307a358a99f0d --- /dev/null +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManagerFactory.java @@ -0,0 +1,74 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sidecar.functionNamespace; + +import com.facebook.airlift.bootstrap.Bootstrap; +import com.facebook.presto.functionNamespace.execution.NoopSqlFunctionExecutorsModule; +import com.facebook.presto.sidecar.NativeSidecarCommunicationModule; +import com.facebook.presto.sidecar.NativeSidecarPlugin; +import com.facebook.presto.spi.function.FunctionHandleResolver; +import com.facebook.presto.spi.function.FunctionNamespaceManager; +import com.facebook.presto.spi.function.FunctionNamespaceManagerContext; +import com.facebook.presto.spi.function.FunctionNamespaceManagerFactory; +import com.facebook.presto.spi.function.SqlFunctionHandle; +import com.google.inject.Injector; + +import java.util.Map; + +import static com.google.common.base.Throwables.throwIfUnchecked; + +/** + * Factory class to create instance of {@link NativeFunctionNamespaceManager}. + * This factor is registered in {@link NativeSidecarPlugin#getFunctionNamespaceManagerFactories()}. + */ +public class NativeFunctionNamespaceManagerFactory + implements FunctionNamespaceManagerFactory +{ + public static final String NAME = "native"; + + private static final SqlFunctionHandle.Resolver HANDLE_RESOLVER = new SqlFunctionHandle.Resolver(); + + @Override + public String getName() + { + return NAME; + } + + @Override + public FunctionHandleResolver getHandleResolver() + { + return HANDLE_RESOLVER; + } + + @Override + public FunctionNamespaceManager create(String catalogName, Map config, FunctionNamespaceManagerContext context) + { + try { + Bootstrap app = new Bootstrap( + new NativeFunctionNamespaceManagerModule(catalogName, context.getNodeManager(), context.getFunctionMetadataManager()), + new NoopSqlFunctionExecutorsModule(), + new NativeSidecarCommunicationModule()); + + Injector injector = app + .doNotInitializeLogging() + .setRequiredConfigurationProperties(config) + .initialize(); + return injector.getInstance(NativeFunctionNamespaceManager.class); + } + catch (Exception e) { + throwIfUnchecked(e); + throw new RuntimeException(e); + } + } +} diff --git a/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManagerModule.java b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManagerModule.java new file mode 100644 index 0000000000000..50f36269456d3 --- /dev/null +++ b/presto-native-sidecar-plugin/src/main/java/com/facebook/presto/sidecar/functionNamespace/NativeFunctionNamespaceManagerModule.java @@ -0,0 +1,65 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sidecar.functionNamespace; + +import com.facebook.airlift.json.JsonCodec; +import com.facebook.airlift.json.JsonCodecFactory; +import com.facebook.presto.functionNamespace.JsonBasedUdfFunctionMetadata; +import com.facebook.presto.functionNamespace.ServingCatalog; +import com.facebook.presto.functionNamespace.SqlInvokedFunctionNamespaceManagerConfig; +import com.facebook.presto.functionNamespace.execution.SqlFunctionLanguageConfig; +import com.facebook.presto.spi.NodeManager; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.TypeLiteral; + +import java.util.List; +import java.util.Map; + +import static com.facebook.airlift.configuration.ConfigBinder.configBinder; +import static com.facebook.airlift.json.JsonCodec.listJsonCodec; +import static com.google.inject.Scopes.SINGLETON; +import static java.util.Objects.requireNonNull; + +public class NativeFunctionNamespaceManagerModule + implements Module +{ + private final String catalogName; + + private final NodeManager nodeManager; + private final FunctionMetadataManager functionMetadataManager; + + public NativeFunctionNamespaceManagerModule(String catalogName, NodeManager nodeManager, FunctionMetadataManager functionMetadataManager) + { + this.catalogName = requireNonNull(catalogName, "catalogName is null"); + this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); + this.functionMetadataManager = requireNonNull(functionMetadataManager, "functionMetadataManager is null"); + } + + @Override + public void configure(Binder binder) + { + binder.bind(new TypeLiteral() {}).annotatedWith(ServingCatalog.class).toInstance(catalogName); + + configBinder(binder).bindConfig(SqlInvokedFunctionNamespaceManagerConfig.class); + configBinder(binder).bindConfig(SqlFunctionLanguageConfig.class); + binder.bind(FunctionDefinitionProvider.class).to(NativeFunctionDefinitionProvider.class).in(SINGLETON); + binder.bind(new TypeLiteral>>>() {}) + .toInstance(new JsonCodecFactory().mapJsonCodec(String.class, listJsonCodec(JsonBasedUdfFunctionMetadata.class))); + binder.bind(NativeFunctionNamespaceManager.class).in(SINGLETON); + binder.bind(NodeManager.class).toInstance(nodeManager); + binder.bind(FunctionMetadataManager.class).toInstance(functionMetadataManager); + } +} diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/NativeSidecarPluginQueryRunnerUtils.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/NativeSidecarPluginQueryRunnerUtils.java index 4ac96f6f837fc..994d824195600 100644 --- a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/NativeSidecarPluginQueryRunnerUtils.java +++ b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/NativeSidecarPluginQueryRunnerUtils.java @@ -12,8 +12,11 @@ * limitations under the License. */ package com.facebook.presto.sidecar; + +import com.facebook.presto.sidecar.functionNamespace.NativeFunctionNamespaceManagerFactory; import com.facebook.presto.sidecar.sessionpropertyproviders.NativeSystemSessionPropertyProviderFactory; import com.facebook.presto.testing.QueryRunner; +import com.google.common.collect.ImmutableMap; public class NativeSidecarPluginQueryRunnerUtils { @@ -23,5 +26,11 @@ public static void setupNativeSidecarPlugin(QueryRunner queryRunner) { queryRunner.installCoordinatorPlugin(new NativeSidecarPlugin()); queryRunner.loadSessionPropertyProvider(NativeSystemSessionPropertyProviderFactory.NAME); + queryRunner.loadFunctionNamespaceManager( + NativeFunctionNamespaceManagerFactory.NAME, + "native", + ImmutableMap.of( + "supported-function-languages", "CPP", + "function-implementation-type", "CPP")); } } diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestConvertApplicableTypeToVariable.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestConvertApplicableTypeToVariable.java new file mode 100644 index 0000000000000..93f9ee75f2b82 --- /dev/null +++ b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestConvertApplicableTypeToVariable.java @@ -0,0 +1,268 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sidecar; + +import com.facebook.presto.common.type.NamedTypeSignature; +import com.facebook.presto.common.type.TypeSignature; +import com.facebook.presto.common.type.TypeSignatureParameter; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; +import static com.facebook.presto.sidecar.functionNamespace.NativeFunctionNamespaceManager.convertApplicableTypeToVariable; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +public class TestConvertApplicableTypeToVariable +{ + @Test + public void testConvertApplicableTypeToVariableDecimalType() + { + TypeSignature actualTypeSignature = new TypeSignature( + "decimal", + TypeSignatureParameter.of(parseTypeSignature("i4")), + TypeSignatureParameter.of(parseTypeSignature("i5"))); + TypeSignature expectedTypeSignature = new TypeSignature( + "decimal", + TypeSignatureParameter.of("i4"), + TypeSignatureParameter.of("i5")); + for (int i = 0; i < actualTypeSignature.getParameters().size(); i++) { + assertTrue(actualTypeSignature.getParameters().get(i).isTypeSignature()); + assertTrue(expectedTypeSignature.getParameters().get(i).isVariable()); + } + TypeSignature resolvedTypeSignature = convertApplicableTypeToVariable(actualTypeSignature); + assertEquals(expectedTypeSignature, resolvedTypeSignature); + } + + @Test + public void testConvertApplicableTypeToVariableDecimalTypeSignature() + { + TypeSignature actualTypeSignature = parseTypeSignature("decimal(i4, i5)"); + TypeSignature expectedTypeSignature = new TypeSignature( + "decimal", + TypeSignatureParameter.of("i4"), + TypeSignatureParameter.of("i5")); + TypeSignature resolvedTypeSignature = convertApplicableTypeToVariable(actualTypeSignature); + assertEquals(expectedTypeSignature, resolvedTypeSignature); + } + + @Test + public void testConvertApplicableTypeToVariableVarcharType() + { + TypeSignature actualTypeSignature = new TypeSignature( + "varchar", + TypeSignatureParameter.of(50)); // a random number + TypeSignature expectedTypeSignature = new TypeSignature( + "varchar", + TypeSignatureParameter.of(50)); + TypeSignature resolvedTypeSignature = convertApplicableTypeToVariable(actualTypeSignature); + assertEquals(expectedTypeSignature, resolvedTypeSignature); + } + + @Test + public void testConvertApplicableTypeToVariableBigintType() + { + TypeSignature actualTypeSignature = parseTypeSignature("bigint"); + TypeSignature expectedTypeSignature = parseTypeSignature("bigint"); + TypeSignature resolvedTypeSignature = convertApplicableTypeToVariable(actualTypeSignature); + assertEquals(expectedTypeSignature, resolvedTypeSignature); + } + + @Test + public void testConvertApplicableTypeToVariableBigintArgType() + { + TypeSignature actualTypeSignature = parseTypeSignature("test(bigint)"); + TypeSignature expectedTypeSignature = parseTypeSignature("test(bigint)"); + List resolvedTypeSignaturesList = + convertApplicableTypeToVariable(actualTypeSignature.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(expectedTypeSignature.getTypeOrNamedTypeParametersAsTypeSignatures(), resolvedTypeSignaturesList); + } + + @Test + public void testConvertApplicableTypeToVariableArrayArgType() + { + TypeSignature actualTypeSignature = parseTypeSignature("test(array(decimal(i4, i5)))"); + TypeSignature expectedTypeSignature = new TypeSignature( + "test", + TypeSignatureParameter.of( + new TypeSignature( + "array", + TypeSignatureParameter.of( + new TypeSignature( + "decimal", + TypeSignatureParameter.of("i4"), + TypeSignatureParameter.of("i5")))))); + List resolvedTypeSignaturesList = + convertApplicableTypeToVariable(actualTypeSignature.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(expectedTypeSignature.getTypeOrNamedTypeParametersAsTypeSignatures(), resolvedTypeSignaturesList); + } + + @Test + public void testConvertApplicableTypeToVariableArrayConstructor() + { + TypeSignature actualTypeSignature = parseTypeSignature("array_constructor(T)"); + TypeSignature expectedTypeSignature = parseTypeSignature("array_constructor(T)"); + TypeSignature resolvedTypeSignature = convertApplicableTypeToVariable(actualTypeSignature); + assertEquals(expectedTypeSignature, resolvedTypeSignature); + } + + @Test + public void testConvertApplicableTypeToVariableMapArgType() + { + TypeSignature actualTypeSignature = parseTypeSignature("test(map(varchar, decimal(i4, i5)))"); + TypeSignature expectedTypeSignature = new TypeSignature( + "test", + TypeSignatureParameter.of( + new TypeSignature( + "map", + TypeSignatureParameter.of(parseTypeSignature("varchar")), + TypeSignatureParameter.of( + new TypeSignature( + "decimal", + TypeSignatureParameter.of("i4"), + TypeSignatureParameter.of("i5")))))); + List resolvedTypeSignaturesList = + convertApplicableTypeToVariable(actualTypeSignature.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(expectedTypeSignature.getTypeOrNamedTypeParametersAsTypeSignatures(), resolvedTypeSignaturesList); + } + + @Test + public void testConvertApplicableTypeToVariableRowType() + { + TypeSignature actualTypeSignature = parseTypeSignature("test(row(varchar, decimal(i4, i5)))"); + TypeSignature expectedTypeSignature = new TypeSignature( + "test", + TypeSignatureParameter.of( + new TypeSignature( + "row", + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.empty(), + parseTypeSignature("varchar"))), + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.empty(), + new TypeSignature( + "decimal", + TypeSignatureParameter.of("i4"), + TypeSignatureParameter.of("i5"))))))); + List resolvedTypeSignaturesList = + convertApplicableTypeToVariable(actualTypeSignature.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(expectedTypeSignature.getTypeOrNamedTypeParametersAsTypeSignatures(), resolvedTypeSignaturesList); + } + + @Test + public void testConvertApplicableTypeToVariableRowTypeRowArgType() + { + TypeSignature actualTypeSignature = parseTypeSignature("test(row(varchar, decimal(i4, i5), row(bigint, decimal(i4, i5))))"); + TypeSignature expectedTypeSignature = new TypeSignature( + "test", + TypeSignatureParameter.of( + new TypeSignature( + "row", + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.empty(), + parseTypeSignature("varchar"))), + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.empty(), + new TypeSignature( + "decimal", + TypeSignatureParameter.of("i4"), + TypeSignatureParameter.of("i5")))), + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.empty(), + new TypeSignature( + "row", + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.empty(), + parseTypeSignature("bigint"))), + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.empty(), + new TypeSignature( + "decimal", + TypeSignatureParameter.of("i4"), + TypeSignatureParameter.of("i5")))))))))); + List resolvedTypeSignaturesList = + convertApplicableTypeToVariable(actualTypeSignature.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(expectedTypeSignature.getTypeOrNamedTypeParametersAsTypeSignatures(), resolvedTypeSignaturesList); + } + + @Test + public void testConvertApplicableTypeToVariableRowTypeRowComplexArgType() + { + TypeSignature actualTypeSignature = parseTypeSignature("row(array(decimal(i4, i5)), decimal(i4, i5), row(bigint, varchar))"); + TypeSignature expectedTypeSignature = new TypeSignature( + "row", + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.empty(), + new TypeSignature( + "array", + TypeSignatureParameter.of( + new TypeSignature( + "decimal", + TypeSignatureParameter.of("i4"), + TypeSignatureParameter.of("i5")))))), + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.empty(), + new TypeSignature( + "decimal", + TypeSignatureParameter.of("i4"), + TypeSignatureParameter.of("i5")))), + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.empty(), + new TypeSignature( + "row", + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.empty(), + parseTypeSignature("bigint"))), + TypeSignatureParameter.of( + new NamedTypeSignature( + Optional.empty(), + parseTypeSignature("varchar"))))))); + List resolvedTypeSignaturesList = + convertApplicableTypeToVariable(actualTypeSignature.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(expectedTypeSignature.getTypeOrNamedTypeParametersAsTypeSignatures(), resolvedTypeSignaturesList); + } + + @Test + public void testConvertApplicableTypeToVariableGenericArrayParamType() + { + TypeSignature actualTypeSignature = parseTypeSignature("test(array(decimal(i4, i5)), T)"); + TypeSignature expectedTypeSignature = new TypeSignature( + "test", + TypeSignatureParameter.of( + new TypeSignature( + "array", + TypeSignatureParameter.of( + new TypeSignature( + "decimal", + TypeSignatureParameter.of("i4"), + TypeSignatureParameter.of("i5"))))), + TypeSignatureParameter.of(parseTypeSignature("T"))); + List resolvedTypeSignaturesList = + convertApplicableTypeToVariable(actualTypeSignature.getTypeOrNamedTypeParametersAsTypeSignatures()); + assertEquals(expectedTypeSignature.getTypeOrNamedTypeParametersAsTypeSignatures(), resolvedTypeSignaturesList); + } +} diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPlugin.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPlugin.java new file mode 100644 index 0000000000000..ba1d134c1167b --- /dev/null +++ b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPlugin.java @@ -0,0 +1,247 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sidecar; + +import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; +import com.facebook.presto.testing.MaterializedResult; +import com.facebook.presto.testing.MaterializedRow; +import com.facebook.presto.testing.QueryRunner; +import com.facebook.presto.tests.AbstractTestQueryFramework; +import com.facebook.presto.tests.DistributedQueryRunner; +import com.google.common.collect.ImmutableList; +import org.intellij.lang.annotations.Language; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.UUID; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createLineitem; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createNation; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createOrders; +import static com.facebook.presto.nativeworker.NativeQueryRunnerUtils.createOrdersEx; +import static com.facebook.presto.sidecar.NativeSidecarPluginQueryRunnerUtils.setupNativeSidecarPlugin; +import static java.lang.String.format; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.fail; + +@Test(singleThreaded = true) +public class TestNativeSidecarPlugin + extends AbstractTestQueryFramework +{ + private static final String REGEX_FUNCTION_NAMESPACE = "native.default.*"; + private static final String REGEX_SESSION_NAMESPACE = "Native Execution only.*"; + + @Override + protected void createTables() + { + QueryRunner queryRunner = (QueryRunner) getExpectedQueryRunner(); + createLineitem(queryRunner); + createNation(queryRunner); + createOrders(queryRunner); + createOrdersEx(queryRunner); + } + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + DistributedQueryRunner queryRunner = (DistributedQueryRunner) PrestoNativeQueryRunnerUtils.createQueryRunner(true, true); + setupNativeSidecarPlugin(queryRunner); + return queryRunner; + } + + @Override + protected QueryRunner createExpectedQueryRunner() + throws Exception + { + return PrestoNativeQueryRunnerUtils.createJavaQueryRunner(); + } + + @Test + public void testShowSession() + { + @Language("SQL") String sql = "SHOW SESSION"; + MaterializedResult actualResult = computeActual(sql); + List actualRows = actualResult.getMaterializedRows(); + List filteredRows = excludeSystemSessionProperties(actualRows); + assertFalse(filteredRows.isEmpty()); + } + + @Test + public void testSetJavaWorkerSessionProperty() + { + @Language("SQL") String setSession = "SET SESSION aggregation_spill_enabled=false"; + MaterializedResult setSessionResult = computeActual(setSession); + assertEquals( + setSessionResult.toString(), + "MaterializedResult{rows=[[true]], " + + "types=[boolean], " + + "setSessionProperties={aggregation_spill_enabled=false}, " + + "resetSessionProperties=[], updateType=SET SESSION}"); + } + + @Test + public void testSetNativeWorkerSessionProperty() + { + @Language("SQL") String setSession = "SET SESSION driver_cpu_time_slice_limit_ms=500"; + MaterializedResult setSessionResult = computeActual(setSession); + assertEquals( + setSessionResult.toString(), + "MaterializedResult{rows=[[true]], " + + "types=[boolean], " + + "setSessionProperties={driver_cpu_time_slice_limit_ms=500}, " + + "resetSessionProperties=[], updateType=SET SESSION}"); + } + + @Test + public void testShowFunctions() + { + @Language("SQL") String sql = "SHOW FUNCTIONS"; + MaterializedResult actualResult = computeActual(sql); + List actualRows = actualResult.getMaterializedRows(); + for (MaterializedRow actualRow : actualRows) { + List row = actualRow.getFields(); + // No namespace should be present on the functionNames + String functionName = row.get(0).toString(); + if (Pattern.matches(REGEX_FUNCTION_NAMESPACE, functionName)) { + fail(format("Namespace match found for row: %s", row)); + } + + // function namespace should be present. + String fullFunctionName = row.get(5).toString(); + if (Pattern.matches(REGEX_FUNCTION_NAMESPACE, fullFunctionName)) { + continue; + } + fail(format("No namespace match found for row: %s", row)); + } + } + + @Test + public void testGeneralQueries() + { + assertQuery("SELECT substr(comment, 1, 10), length(comment), trim(comment) FROM orders"); + assertQuery("SELECT substr(comment, 1, 10), length(comment), ltrim(comment) FROM orders"); + assertQuery("SELECT substr(comment, 1, 10), length(comment), rtrim(comment) FROM orders"); + assertQuery("select lower(comment) from nation"); + assertQuery("SELECT trim(comment, ' ns'), ltrim(comment, 'a b c'), rtrim(comment, 'l y') FROM orders"); + assertQuery("select array[nationkey], array_constructor(comment) from nation"); + assertQuery("SELECT nationkey, bit_count(nationkey, 10) FROM nation ORDER BY 1"); + assertQuery("SELECT * FROM lineitem WHERE shipinstruct like 'TAKE BACK%'"); + assertQuery("SELECT * FROM lineitem WHERE shipinstruct like 'TAKE BACK#%' escape '#'"); + assertQuery("SELECT orderkey, date_trunc('year', from_unixtime(orderkey, '-03:00')), date_trunc('quarter', from_unixtime(orderkey, '+14:00')), " + + "date_trunc('month', from_unixtime(orderkey, '+03:00')), date_trunc('day', from_unixtime(orderkey, '-07:00')), " + + "date_trunc('hour', from_unixtime(orderkey, '-09:30')), date_trunc('minute', from_unixtime(orderkey, '+05:30')), " + + "date_trunc('second', from_unixtime(orderkey, '+00:00')) FROM orders"); + } + + @Test + public void testAggregateFunctions() + { + assertQuery("select corr(nationkey, nationkey) from nation"); + assertQuery("select count(comment) from orders"); + assertQuery("select count(*) from nation"); + assertQuery("select count(abs(orderkey) between 1 and 60000) from orders group by orderkey"); + assertQuery("SELECT count(orderkey) FROM orders WHERE orderkey < 0 GROUP BY GROUPING SETS (())"); + // tinyint + assertQuery("SELECT sum(cast(linenumber as tinyint)), sum(cast(linenumber as tinyint)) FROM lineitem"); + // smallint + assertQuery("SELECT sum(cast(linenumber as smallint)), sum(cast(linenumber as smallint)) FROM lineitem"); + // integer + assertQuery("SELECT sum(linenumber), sum(linenumber) FROM lineitem"); + // bigint + assertQuery("SELECT sum(orderkey), sum(orderkey) FROM lineitem"); + // real + assertQuery("SELECT sum(tax_as_real), sum(tax_as_real) FROM lineitem"); + // double + assertQuery("SELECT sum(quantity), sum(quantity) FROM lineitem"); + // date + assertQuery("SELECT approx_distinct(orderdate, 0.023) FROM orders"); + // timestamp + assertQuery("SELECT approx_distinct(CAST(orderdate AS TIMESTAMP)) FROM orders"); + assertQuery("SELECT approx_distinct(CAST(orderdate AS TIMESTAMP), 0.023) FROM orders"); + assertQuery("SELECT checksum(from_unixtime(orderkey, '+01:00')) FROM lineitem WHERE orderkey < 20"); + assertQuerySucceeds("SELECT shuffle(array_sort(quantities)) FROM orders_ex"); + assertQuery("SELECT array_sort(shuffle(quantities)) FROM orders_ex"); + } + + @Test + public void testWindowFunctions() + { + assertQuery("SELECT * FROM (SELECT row_number() over(partition by orderstatus order by orderkey, orderstatus) rn, * from orders) WHERE rn = 1"); + assertQuery("WITH t AS (SELECT linenumber, row_number() over (partition by linenumber order by linenumber) as rn FROM lineitem) SELECT * FROM t WHERE rn = 1"); + assertQuery("SELECT row_number() OVER (PARTITION BY orderdate ORDER BY orderdate) FROM orders"); + assertQuery("SELECT min(orderkey) OVER (PARTITION BY orderdate ORDER BY orderdate, totalprice) FROM orders"); + assertQuery("SELECT sum(rn) FROM (SELECT row_number() over() rn, * from orders) WHERE rn = 10"); + assertQuery("SELECT * FROM (SELECT row_number() over(partition by orderstatus order by orderkey) rn, * from orders) WHERE rn = 1"); + } + + @Test + public void testArraySort() + { + assertQueryFails("SELECT array_sort(quantities, (x, y) -> if (x < y, 1, if (x > y, -1, 0))) FROM orders_ex", + "line 1:31: Expected a lambda that takes 1 argument\\(s\\) but got 2"); + assertQueryFails("SELECT orderkey, array_sort(reduce_agg(linenumber, CAST(array[] as ARRAY(INTEGER)), (s, x) -> s || x, (s, s2) -> s || s2)) FROM lineitem group by orderkey", + ".*Unexpected parameters \\(array\\(integer\\), array\\(integer\\)\\) for function native.default.concat.*"); + } + + @Test + public void testInformationSchemaTables() + { + assertQueryFails("select lower(table_name) from information_schema.tables " + + "where table_name = 'lineitem' or table_name = 'LINEITEM' ", + "Compiler failed"); + } + + @Test + public void testShowStats() + { + String tmpTableName = generateRandomTableName(); + try { + getQueryRunner().execute(String.format("CREATE TABLE %s (c0 DECIMAL(15,2), c1 DECIMAL(38,2)) WITH (format = 'PARQUET')", tmpTableName)); + getQueryRunner().execute(String.format("INSERT INTO %s VALUES (DECIMAL '0', DECIMAL '0'), (DECIMAL '1.2', DECIMAL '3.4'), " + + "(DECIMAL '1000000.12', DECIMAL '28239823232323.57'), " + + "(DECIMAL '-542392.89', DECIMAL '-6723982392109.29'), (NULL, NULL), " + + "(NULL, DECIMAL'-6723982392109.29'),(DECIMAL'1.2', NULL)", tmpTableName)); + assertQuery(String.format("SHOW STATS for %s", tmpTableName)); + } + finally { + dropTableIfExists(tmpTableName); + } + } + + private String generateRandomTableName() + { + String tableName = "tmp_presto_" + UUID.randomUUID().toString().replace("-", ""); + // Clean up if the temporary named table already exists. + dropTableIfExists(tableName); + return tableName; + } + + private void dropTableIfExists(String tableName) + { + // An ugly workaround for the lack of getExpectedQueryRunner() + computeExpected(String.format("DROP TABLE IF EXISTS %s", tableName), ImmutableList.of(BIGINT)); + } + + private List excludeSystemSessionProperties(List inputRows) + { + return inputRows.stream() + .filter(row -> Pattern.matches(REGEX_SESSION_NAMESPACE, row.getFields().get(4).toString())) + .collect(Collectors.toList()); + } +} diff --git a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPluginSystemPropertyProvider.java b/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPluginSystemPropertyProvider.java deleted file mode 100644 index 0ea9eec5b49aa..0000000000000 --- a/presto-native-sidecar-plugin/src/test/java/com/facebook/presto/sidecar/TestNativeSidecarPluginSystemPropertyProvider.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.sidecar; - -import com.facebook.presto.nativeworker.PrestoNativeQueryRunnerUtils; -import com.facebook.presto.testing.MaterializedResult; -import com.facebook.presto.testing.MaterializedRow; -import com.facebook.presto.testing.QueryRunner; -import com.facebook.presto.tests.AbstractTestQueryFramework; -import com.facebook.presto.tests.DistributedQueryRunner; -import org.intellij.lang.annotations.Language; -import org.testng.annotations.Test; - -import java.util.List; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import static com.facebook.presto.sidecar.NativeSidecarPluginQueryRunnerUtils.setupNativeSidecarPlugin; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertFalse; - -@Test(singleThreaded = true) -public class TestNativeSidecarPluginSystemPropertyProvider - extends AbstractTestQueryFramework -{ - private static final String REGEX_SESSION_NAMESPACE = "Native Execution only.*"; - - @Override - protected QueryRunner createQueryRunner() - throws Exception - { - DistributedQueryRunner queryRunner = (DistributedQueryRunner) PrestoNativeQueryRunnerUtils.createQueryRunner(false, true); - setupNativeSidecarPlugin(queryRunner); - return queryRunner; - } - - @Override - protected QueryRunner createExpectedQueryRunner() - throws Exception - { - return PrestoNativeQueryRunnerUtils.createJavaQueryRunner(); - } - - private List excludeSystemSessionProperties(List inputRows) - { - return inputRows.stream() - .filter(row -> Pattern.matches(REGEX_SESSION_NAMESPACE, row.getFields().get(4).toString())) - .collect(Collectors.toList()); - } - - @Test - public void testShowSession() - { - @Language("SQL") String sql = "SHOW SESSION"; - MaterializedResult actualResult = computeActual(sql); - List actualRows = actualResult.getMaterializedRows(); - List filteredRows = excludeSystemSessionProperties(actualRows); - assertFalse(filteredRows.isEmpty()); - } - - @Test - public void testSetJavaWorkerSessionProperty() - { - @Language("SQL") String setSession = "SET SESSION aggregation_spill_enabled=false"; - MaterializedResult setSessionResult = computeActual(setSession); - assertEquals( - setSessionResult.toString(), - "MaterializedResult{rows=[[true]], " + - "types=[boolean], " + - "setSessionProperties={aggregation_spill_enabled=false}, " + - "resetSessionProperties=[], updateType=SET SESSION}"); - } - - @Test - public void testSetNativeWorkerSessionProperty() - { - @Language("SQL") String setSession = "SET SESSION driver_cpu_time_slice_limit_ms=500"; - MaterializedResult setSessionResult = computeActual(setSession); - assertEquals( - setSessionResult.toString(), - "MaterializedResult{rows=[[true]], " + - "types=[boolean], " + - "setSessionProperties={driver_cpu_time_slice_limit_ms=500}, " + - "resetSessionProperties=[], updateType=SET SESSION}"); - } -} diff --git a/presto-openapi/src/main/resources/function_signatures.yaml b/presto-openapi/src/main/resources/function_signatures.yaml new file mode 100644 index 0000000000000..7871a2960b4a8 --- /dev/null +++ b/presto-openapi/src/main/resources/function_signatures.yaml @@ -0,0 +1,20 @@ +openapi: 3.0.0 +info: + title: Presto function signatures API + description: API for retrieving function signatures in Presto. + version: "1" +servers: + - url: http://localhost:8080 + description: Presto endpoint when running locally +paths: + /v1/functions: + get: + summary: Returns the list of function signatures. + description: This endpoint retrieves the supported function signatures from a Prestissimo cluster. + responses: + '200': + description: List of function signatures. + content: + application/json: + schema: + $ref: './schemas.yaml/#/components/schemas/JsonBasedUdfFunctionMetadata' diff --git a/presto-openapi/src/main/resources/schemas.yaml b/presto-openapi/src/main/resources/schemas.yaml index b9f3c312bc4fc..bcb5aaca438c9 100644 --- a/presto-openapi/src/main/resources/schemas.yaml +++ b/presto-openapi/src/main/resources/schemas.yaml @@ -1018,6 +1018,60 @@ components: type: string hidden: type: boolean + AggregationFunctionMetadata: + type: object + required: + - intermediateType + - isOrderSensitive + properties: + intermediateType: + $ref: '#/components/schemas/TypeSignature' + description: Intermediate TypeSignature for the aggregation function + isOrderSensitive: + type: boolean + description: Determines if the corresponding aggregation function is order-sensitive + JsonBasedUdfFunctionMetadata: + type: object + required: + - docString + - functionKind + - outputType + - paramTypes + - schema + - routineCharacteristics + - variableArity + properties: + docString: + type: string + description: Description of the function + functionKind: + $ref: '#/components/schemas/FunctionKind' + description: FunctionKind of the function + outputType: + $ref: '#/components/schemas/TypeSignature' + description: Output type of the function + paramTypes: + type: array + items: + $ref: '#/components/schemas/TypeSignature' + description: Input types of the function + schema: + type: string + description: Schema the function belongs to. Catalog.schema.function uniquely identifies a function. + routineCharacteristics: + $ref: '#/components/schemas/RoutineCharacteristics' + description: Implement language of the function. + variableArity: + type: boolean + description: Arity of the function. + typeVariableConstraints: + type: array + items: + $ref: '#/components/schemas/TypeVariableConstraint' + description: Optional list of the typeVariableConstraints. + aggregationFunctionMetadata: + $ref: '#/components/schemas/AggregationFunctionMetadata' + description: Optional Aggregate-specific metadata (required for aggregation functions) diff --git a/presto-spark-base/pom.xml b/presto-spark-base/pom.xml index 8ff638fdc742f..d73651fbdb800 100644 --- a/presto-spark-base/pom.xml +++ b/presto-spark-base/pom.xml @@ -250,6 +250,13 @@ + + com.facebook.presto + presto-function-namespace-managers-common + ${project.version} + test + + com.facebook.presto presto-hive-metastore diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java index e3a3e0e8dfbd6..e9d03e52a461f 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/PrestoSparkModule.java @@ -134,6 +134,7 @@ import com.facebook.presto.spark.planner.optimizers.AdaptivePlanOptimizers; import com.facebook.presto.spi.ConnectorMetadataUpdateHandle; import com.facebook.presto.spi.ConnectorTypeSerde; +import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.PageIndexerFactory; import com.facebook.presto.spi.PageSorter; import com.facebook.presto.spi.analyzer.ViewDefinition; @@ -514,7 +515,7 @@ protected void setup(Binder binder) // TODO: Decouple and remove: required by ConnectorManager binder.bind(InternalNodeManager.class).toInstance(new PrestoSparkInternalNodeManager()); - binder.bind(PluginNodeManager.class); + binder.bind(PluginNodeManager.class).in(Scopes.SINGLETON); // TODO: Decouple and remove: required by PluginManager binder.bind(InternalResourceGroupManager.class).in(Scopes.SINGLETON); @@ -561,6 +562,7 @@ protected void setup(Binder binder) newSetBinder(binder, PrestoSparkAuthenticatorProvider.class); binder.bind(ClientRequestFilterManager.class).in(Scopes.SINGLETON); binder.bind(PlanCheckerProviderManager.class).in(Scopes.SINGLETON); + binder.bind(NodeManager.class).to(PluginNodeManager.class).in(Scopes.SINGLETON); } @Provides diff --git a/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java b/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java index 2ddde52dfa793..c5adcf7f0efe2 100644 --- a/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java +++ b/presto-spark-base/src/test/java/com/facebook/presto/spark/PrestoSparkQueryRunner.java @@ -44,6 +44,7 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.MetadataUtil; import com.facebook.presto.metadata.SessionPropertyManager; +import com.facebook.presto.nodeManager.PluginNodeManager; import com.facebook.presto.server.PluginManager; import com.facebook.presto.spark.accesscontrol.PrestoSparkAccessControlCheckerExecution; import com.facebook.presto.spark.classloader_interface.ExecutionStrategy; @@ -57,6 +58,7 @@ import com.facebook.presto.spark.classloader_interface.PrestoSparkTaskExecutorFactoryProvider; import com.facebook.presto.spark.execution.AbstractPrestoSparkQueryExecution; import com.facebook.presto.spark.execution.nativeprocess.NativeExecutionModule; +import com.facebook.presto.spi.NodeManager; import com.facebook.presto.spi.Plugin; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.eventlistener.EventListener; @@ -175,6 +177,8 @@ public class PrestoSparkQueryRunner private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private final NodeManager nodeManager; + protected static final MetastoreContext METASTORE_CONTEXT = new MetastoreContext("test_user", "test_queryId", Optional.empty(), Collections.emptySet(), Optional.empty(), Optional.empty(), false, HiveColumnConverterProvider.DEFAULT_COLUMN_CONVERTER_PROVIDER, WarningCollector.NOOP, new RuntimeStats()); public static PrestoSparkQueryRunner createHivePrestoSparkQueryRunner() @@ -346,6 +350,7 @@ public PrestoSparkQueryRunner( sparkContext = sparkContextHolder.get(additionalSparkProperties, availableCpuCount); prestoSparkService = injector.getInstance(PrestoSparkService.class); testingAccessControlManager = injector.getInstance(TestingAccessControlManager.class); + nodeManager = injector.getInstance(PluginNodeManager.class); // Install tpch Plugin pluginManager.installPlugin(new TpchPlugin()); @@ -678,7 +683,7 @@ public void createCatalog(String catalogName, String connectorName, Map properties) { - metadata.getFunctionAndTypeManager().loadFunctionNamespaceManager(functionNamespaceManagerName, catalogName, properties); + metadata.getFunctionAndTypeManager().loadFunctionNamespaceManager(functionNamespaceManagerName, catalogName, properties, nodeManager); } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java b/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java index 894a8abea5120..09aa55cb24111 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java @@ -141,6 +141,7 @@ public enum StandardErrorCode TOO_MANY_SIDECARS(0x0002_0013, INTERNAL_ERROR), NO_CPP_SIDECARS(0x0002_0014, INTERNAL_ERROR), HEADER_MODIFICATION_ATTEMPT(0x0002_0015, INTERNAL_ERROR), + DUPLICATE_FUNCTION_ERROR(0x0002_0016, INTERNAL_ERROR), /**/; // Error code range 0x0003 is reserved for Presto-on-Spark diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandle.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandle.java index 7cb5f9ab375b4..f7e9bc46ff07a 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandle.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionHandle.java @@ -14,6 +14,9 @@ package com.facebook.presto.spi.function; import com.facebook.presto.common.CatalogSchemaName; +import com.facebook.presto.common.type.TypeSignature; + +import java.util.List; /** * FunctionHandle is a unique handle to identify the function implementation from namespaces. @@ -26,4 +29,6 @@ public interface FunctionHandle String getName(); FunctionKind getKind(); + + List getArgumentTypes(); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadataManager.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadataManager.java index 04231b2c15557..d163ef952e64c 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadataManager.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionMetadataManager.java @@ -16,4 +16,6 @@ public interface FunctionMetadataManager { FunctionMetadata getFunctionMetadata(FunctionHandle functionHandle); + + SqlFunctionSupplier getSpecializedFunctionKey(Signature signature); } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionNamespaceManagerContext.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionNamespaceManagerContext.java index f3ccdbbbcc8dd..92c9ba73853ea 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionNamespaceManagerContext.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/FunctionNamespaceManagerContext.java @@ -14,20 +14,38 @@ package com.facebook.presto.spi.function; import com.facebook.presto.common.type.TypeManager; +import com.facebook.presto.spi.NodeManager; import static java.util.Objects.requireNonNull; public class FunctionNamespaceManagerContext { private final TypeManager typeManager; + private final NodeManager nodeManager; + private final FunctionMetadataManager functionMetadataManager; - public FunctionNamespaceManagerContext(TypeManager typeManager) + public FunctionNamespaceManagerContext( + TypeManager typeManager, + NodeManager nodeManager, + FunctionMetadataManager functionMetadataManager) { this.typeManager = requireNonNull(typeManager, "typeManager is null"); + this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); + this.functionMetadataManager = requireNonNull(functionMetadataManager, "functionMetadataManager is null"); } public TypeManager getTypeManager() { return typeManager; } + + public NodeManager getNodeManager() + { + return nodeManager; + } + + public FunctionMetadataManager getFunctionMetadataManager() + { + return functionMetadataManager; + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionHandle.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionHandle.java index 1982ecbd5c566..132362ba7fe48 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionHandle.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionHandle.java @@ -14,10 +14,12 @@ package com.facebook.presto.spi.function; import com.facebook.presto.common.CatalogSchemaName; +import com.facebook.presto.common.type.TypeSignature; import com.facebook.presto.spi.api.Experimental; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; import java.util.Objects; import static com.facebook.presto.spi.function.FunctionKind.SCALAR; @@ -69,6 +71,12 @@ public FunctionKind getKind() return SCALAR; } + @Override + public List getArgumentTypes() + { + return functionId.getArgumentTypes(); + } + @Override public boolean equals(Object obj) { diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionSupplier.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionSupplier.java new file mode 100644 index 0000000000000..fe3f8988e0548 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlFunctionSupplier.java @@ -0,0 +1,19 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.function; + +public interface SqlFunctionSupplier +{ + SqlFunction getFunction(); +} diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedAggregationFunctionImplementation.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedAggregationFunctionImplementation.java index 6f329a4d6081f..2386caba550dd 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedAggregationFunctionImplementation.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedAggregationFunctionImplementation.java @@ -15,12 +15,19 @@ import com.facebook.presto.common.type.Type; import com.facebook.presto.spi.api.Experimental; +import com.facebook.presto.spi.function.aggregation.Accumulator; +import com.facebook.presto.spi.function.aggregation.AggregationMetadata; +import com.facebook.presto.spi.function.aggregation.GroupedAccumulator; + +import java.util.List; import static java.util.Objects.requireNonNull; +// A no-op implementation of JavaAggregationFunctionImplementation. +// A hacky way to handle lambda aggregate functions which require an implementation of JavaAggregationFunctionImplementation. @Experimental public class SqlInvokedAggregationFunctionImplementation - implements AggregationFunctionImplementation + implements JavaAggregationFunctionImplementation { private final Type intermediateType; @@ -28,11 +35,18 @@ public class SqlInvokedAggregationFunctionImplementation private final boolean isOrderSensitive; - public SqlInvokedAggregationFunctionImplementation(Type intermediateType, Type finalType, boolean isOrderSensitive) + private final List parameterTypes; + + public SqlInvokedAggregationFunctionImplementation( + Type intermediateType, + Type finalType, + boolean isOrderSensitive, + List parameterTypes) { this.intermediateType = requireNonNull(intermediateType, "intermediateType is null"); this.finalType = requireNonNull(finalType, "finalType is null"); this.isOrderSensitive = isOrderSensitive; + this.parameterTypes = requireNonNull(parameterTypes, "parameterTypes is null"); } @Override @@ -59,4 +73,28 @@ public boolean isOrderSensitive() { return isOrderSensitive; } + + @Override + public List getParameterTypes() + { + return parameterTypes; + } + + @Override + public AggregationMetadata getAggregationMetadata() + { + throw new UnsupportedOperationException(); + } + + @Override + public Class getAccumulatorClass() + { + throw new UnsupportedOperationException(); + } + + @Override + public Class getGroupedAccumulatorClass() + { + throw new UnsupportedOperationException(); + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedFunction.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedFunction.java index 15cbf906594e6..4f777c747cd61 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedFunction.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/SqlInvokedFunction.java @@ -47,7 +47,7 @@ public class SqlInvokedFunction private final String description; private final RoutineCharacteristics routineCharacteristics; private final String body; - + private final boolean variableArity; private final Signature signature; private final SqlFunctionId functionId; private final FunctionVersion functionVersion; @@ -65,6 +65,7 @@ public SqlInvokedFunction( @JsonProperty("description") String description, @JsonProperty("routineCharacteristics") RoutineCharacteristics routineCharacteristics, @JsonProperty("body") String body, + @JsonProperty("variableArity") boolean variableArity, @JsonProperty("signature") Signature signature, @JsonProperty("functionId") SqlFunctionId functionId) { @@ -73,6 +74,7 @@ public SqlInvokedFunction( this.routineCharacteristics = routineCharacteristics; this.body = body; this.signature = signature; + this.variableArity = variableArity; this.functionId = functionId; this.functionVersion = notVersioned(); this.functionHandle = Optional.empty(); @@ -89,7 +91,7 @@ public SqlInvokedFunction( String body, FunctionVersion version) { - this(functionName, parameters, emptyList(), returnType, description, routineCharacteristics, body, version, SCALAR, Optional.empty()); + this(functionName, parameters, emptyList(), returnType, description, routineCharacteristics, body, false, version, SCALAR, Optional.empty()); } public SqlInvokedFunction( @@ -103,7 +105,7 @@ public SqlInvokedFunction( FunctionKind kind, Optional aggregationMetadata) { - this(functionName, parameters, emptyList(), returnType, description, routineCharacteristics, body, version, kind, aggregationMetadata); + this(functionName, parameters, emptyList(), returnType, description, routineCharacteristics, body, false, version, kind, aggregationMetadata); } public SqlInvokedFunction( @@ -114,6 +116,7 @@ public SqlInvokedFunction( String description, RoutineCharacteristics routineCharacteristics, String body, + boolean variableArity, FunctionVersion version, FunctionKind kind, Optional aggregationMetadata) @@ -126,6 +129,7 @@ public SqlInvokedFunction( description, routineCharacteristics, body, + variableArity, version, kind, new SqlFunctionId(functionName, getArgumentTypes(parameters)), @@ -145,6 +149,7 @@ public SqlInvokedFunction( String description, RoutineCharacteristics routineCharacteristics, String body, + boolean variableArity, FunctionVersion version, FunctionKind kind, SqlFunctionId functionId, @@ -157,6 +162,7 @@ public SqlInvokedFunction( this.body = requireNonNull(body, "body is null"); this.signature = new Signature(functionName, kind, typeVariableConstraints, emptyList(), returnType, getArgumentTypes(parameters), false); this.functionId = requireNonNull(functionId, "functionId is null"); + this.variableArity = variableArity; this.functionVersion = requireNonNull(version, "version is null"); this.functionHandle = requireNonNull(functionHandle, "functionHandle is null"); this.aggregationMetadata = requireNonNull(aggregationMetadata, "aggregationMetadata is null"); @@ -186,10 +192,12 @@ public SqlInvokedFunction withVersion(String version) return new SqlInvokedFunction( signature.getName(), parameters, + signature.getTypeVariableConstraints(), signature.getReturnType(), description, routineCharacteristics, body, + variableArity, FunctionVersion.withVersion(version), signature.getKind(), aggregationMetadata); @@ -272,6 +280,11 @@ public FunctionVersion getVersion() return functionVersion; } + public boolean getVariableArity() + { + return variableArity; + } + public Optional getAggregationMetadata() { return aggregationMetadata; @@ -302,6 +315,7 @@ public boolean hasSameDefinitionAs(SqlInvokedFunction function) && Objects.equals(description, function.description) && Objects.equals(routineCharacteristics, function.routineCharacteristics) && Objects.equals(body, function.body) + && Objects.equals(variableArity, function.variableArity) && Objects.equals(signature, function.signature) && Objects.equals(aggregationMetadata, function.aggregationMetadata); } @@ -320,6 +334,7 @@ public boolean equals(Object obj) && Objects.equals(description, o.description) && Objects.equals(routineCharacteristics, o.routineCharacteristics) && Objects.equals(body, o.body) + && Objects.equals(variableArity, o.variableArity) && Objects.equals(signature, o.signature) && Objects.equals(functionId, o.functionId) && Objects.equals(functionHandle, o.functionHandle) @@ -329,7 +344,7 @@ public boolean equals(Object obj) @Override public int hashCode() { - return Objects.hash(parameters, description, routineCharacteristics, body, signature, functionId, functionHandle); + return Objects.hash(parameters, description, routineCharacteristics, body, variableArity, signature, functionId, functionHandle); } @Override @@ -346,6 +361,7 @@ public String toString() signature.getKind(), signature.getKind() == AGGREGATE ? ", " + getAggregationMetadata().get() : "", body, - routineCharacteristics); + routineCharacteristics, + variableArity); } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/function/TypeVariableConstraint.java b/presto-spi/src/main/java/com/facebook/presto/spi/function/TypeVariableConstraint.java index 34cb4c1a7eea8..4eb17c63e3e57 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/function/TypeVariableConstraint.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/function/TypeVariableConstraint.java @@ -48,7 +48,7 @@ public TypeVariableConstraint( this.name = name; this.comparableRequired = comparableRequired; this.orderableRequired = orderableRequired; - this.variadicBound = variadicBound; + this.variadicBound = (Objects.equals(variadicBound, "")) ? null : variadicBound; this.nonDecimalNumericRequired = nonDecimalNumericRequired; } diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 107c9f4a09f8b..fa38be6ffb751 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -77,6 +77,12 @@ + + com.facebook.presto + presto-function-namespace-managers-common + ${project.version} + + com.facebook.airlift bootstrap diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java index 5674d59e752b7..b3f25630cb078 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/DistributedQueryRunner.java @@ -740,7 +740,7 @@ public void createCatalog(String catalogName, String connectorName, Map properties) { for (TestingPrestoServer server : servers) { - server.getMetadata().getFunctionAndTypeManager().loadFunctionNamespaceManager(functionNamespaceManagerName, catalogName, properties); + server.getMetadata().getFunctionAndTypeManager().loadFunctionNamespaceManager(functionNamespaceManagerName, catalogName, properties, server.getPluginNodeManager()); } } @@ -955,7 +955,7 @@ private void loadFunctionNamespaceManager( if (coordinatorOnly && !server.isCoordinator()) { continue; } - server.getMetadata().getFunctionAndTypeManager().loadFunctionNamespaceManager(functionNamespaceManagerName, catalogName, properties); + server.getMetadata().getFunctionAndTypeManager().loadFunctionNamespaceManager(functionNamespaceManagerName, catalogName, properties, server.getPluginNodeManager()); } } diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java b/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java index 00993ae054951..9637d82a38683 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/StandaloneQueryRunner.java @@ -254,7 +254,7 @@ public void createCatalog(String catalogName, String connectorName, Map properties) { - server.getMetadata().getFunctionAndTypeManager().loadFunctionNamespaceManager(functionNamespaceManagerName, catalogName, properties); + server.getMetadata().getFunctionAndTypeManager().loadFunctionNamespaceManager(functionNamespaceManagerName, catalogName, properties, server.getPluginNodeManager()); } @Override diff --git a/presto-tests/src/main/java/com/facebook/presto/tests/StatefulSleepingSum.java b/presto-tests/src/main/java/com/facebook/presto/tests/StatefulSleepingSum.java index b0aede8bed133..b1ebaf04daeff 100644 --- a/presto-tests/src/main/java/com/facebook/presto/tests/StatefulSleepingSum.java +++ b/presto-tests/src/main/java/com/facebook/presto/tests/StatefulSleepingSum.java @@ -28,7 +28,7 @@ import java.util.concurrent.atomic.AtomicInteger; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.RETURN_NULL_ON_NULL; import static com.facebook.presto.spi.function.Signature.typeVariable; @@ -46,7 +46,7 @@ public class StatefulSleepingSum private StatefulSleepingSum() { super(new Signature( - QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "stateful_sleeping_sum"), + QualifiedObjectName.valueOf(JAVA_BUILTIN_NAMESPACE, "stateful_sleeping_sum"), FunctionKind.SCALAR, ImmutableList.of(typeVariable("bigint")), ImmutableList.of(), diff --git a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/IgnoredFunctionsMismatchResolver.java b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/IgnoredFunctionsMismatchResolver.java index a3424efac2e80..4a75408ec6bea 100644 --- a/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/IgnoredFunctionsMismatchResolver.java +++ b/presto-verifier/src/main/java/com/facebook/presto/verifier/resolver/IgnoredFunctionsMismatchResolver.java @@ -29,7 +29,7 @@ import java.util.Optional; import java.util.Set; -import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.DEFAULT_NAMESPACE; +import static com.facebook.presto.metadata.BuiltInTypeAndFunctionNamespaceManager.JAVA_BUILTIN_NAMESPACE; import static com.facebook.presto.verifier.framework.DataMatchResult.MatchType.COLUMN_MISMATCH; import static com.facebook.presto.verifier.framework.DataMatchResult.MatchType.ROW_COUNT_MISMATCH; import static com.google.common.base.Preconditions.checkArgument; @@ -93,7 +93,7 @@ private static String normalizeFunctionName(String name) private static String normalizeFunctionName(QualifiedName name) { if (name.getParts().size() == 3 && - new CatalogSchemaName(name.getParts().get(0), name.getParts().get(1)).equals(DEFAULT_NAMESPACE)) { + new CatalogSchemaName(name.getParts().get(0), name.getParts().get(1)).equals(JAVA_BUILTIN_NAMESPACE)) { return name.getSuffix(); } return name.toString();