From f98f715a9a33c2ae13b96480ec3b90e228969bd3 Mon Sep 17 00:00:00 2001 From: kasiafi <30203062+kasiafi@users.noreply.github.com> Date: Tue, 9 May 2023 20:50:49 +0200 Subject: [PATCH] Analyze JSON_TABLE invocation --- .../java/io/trino/sql/analyzer/Analysis.java | 55 +- .../sql/analyzer/ExpressionAnalyzer.java | 224 ++++++-- .../trino/sql/analyzer/JsonPathAnalyzer.java | 10 +- .../trino/sql/analyzer/StatementAnalyzer.java | 295 +++++++++- .../io/trino/sql/planner/RelationPlanner.java | 7 + .../planner/ResolvedFunctionCallRewriter.java | 7 +- .../io/trino/sql/analyzer/TestAnalyzer.java | 539 +++++++++++++++++- .../io/trino/sql/jsonpath/PathParser.java | 25 +- .../io/trino/sql/jsonpath/TestPathParser.java | 2 +- .../java/io/trino/spi/StandardErrorCode.java | 3 + 10 files changed, 1109 insertions(+), 58 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/Analysis.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/Analysis.java index ae373edf3a25..cc82267f610b 100644 --- a/core/trino-main/src/main/java/io/trino/sql/analyzer/Analysis.java +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/Analysis.java @@ -62,6 +62,8 @@ import io.trino.sql.tree.Identifier; import io.trino.sql.tree.InPredicate; import io.trino.sql.tree.Join; +import io.trino.sql.tree.JsonTable; +import io.trino.sql.tree.JsonTableColumnDefinition; import io.trino.sql.tree.LambdaArgumentDeclaration; import io.trino.sql.tree.MeasureDefinition; import io.trino.sql.tree.Node; @@ -159,9 +161,10 @@ public class Analysis private final Set> patternAggregations = new LinkedHashSet<>(); // for JSON features - private final Map, JsonPathAnalysis> jsonPathAnalyses = new LinkedHashMap<>(); + private final Map, JsonPathAnalysis> jsonPathAnalyses = new LinkedHashMap<>(); private final Map, ResolvedFunction> jsonInputFunctions = new LinkedHashMap<>(); - private final Map, ResolvedFunction> jsonOutputFunctions = new LinkedHashMap<>(); + private final Map, ResolvedFunction> jsonOutputFunctions = new LinkedHashMap<>(); + private final Map, JsonTableAnalysis> jsonTableAnalyses = new LinkedHashMap<>(); private final Map, List> aggregates = new LinkedHashMap<>(); private final Map, List> orderByAggregates = new LinkedHashMap<>(); @@ -202,7 +205,7 @@ public class Analysis private final Map, Type> sortKeyCoercionsForFrameBoundComparison = new LinkedHashMap<>(); private final Map, ResolvedFunction> frameBoundCalculations = new LinkedHashMap<>(); private final Map, List> relationCoercions = new LinkedHashMap<>(); - private final Map, RoutineEntry> resolvedFunctions = new LinkedHashMap<>(); + private final Map, RoutineEntry> resolvedFunctions = new LinkedHashMap<>(); private final Map, LambdaArgumentDeclaration> lambdaArgumentReferences = new LinkedHashMap<>(); private final Map columns = new LinkedHashMap<>(); @@ -647,12 +650,12 @@ public void registerTable( columnMaskScopes.isEmpty())); } - public ResolvedFunction getResolvedFunction(Expression node) + public ResolvedFunction getResolvedFunction(Node node) { return resolvedFunctions.get(NodeRef.of(node)).getFunction(); } - public void addResolvedFunction(Expression node, ResolvedFunction function, String authorization) + public void addResolvedFunction(Node node, ResolvedFunction function, String authorization) { resolvedFunctions.put(NodeRef.of(node), new RoutineEntry(function, authorization)); } @@ -1007,14 +1010,19 @@ public boolean isPatternAggregation(FunctionCall function) return patternAggregations.contains(NodeRef.of(function)); } - public void setJsonPathAnalyses(Map, JsonPathAnalysis> pathAnalyses) + public void setJsonPathAnalyses(Map, JsonPathAnalysis> pathAnalyses) { jsonPathAnalyses.putAll(pathAnalyses); } - public JsonPathAnalysis getJsonPathAnalysis(Expression expression) + public void setJsonPathAnalysis(Node node, JsonPathAnalysis pathAnalysis) { - return jsonPathAnalyses.get(NodeRef.of(expression)); + jsonPathAnalyses.put(NodeRef.of(node), pathAnalysis); + } + + public JsonPathAnalysis getJsonPathAnalysis(Node node) + { + return jsonPathAnalyses.get(NodeRef.of(node)); } public void setJsonInputFunctions(Map, ResolvedFunction> functions) @@ -1027,14 +1035,24 @@ public ResolvedFunction getJsonInputFunction(Expression expression) return jsonInputFunctions.get(NodeRef.of(expression)); } - public void setJsonOutputFunctions(Map, ResolvedFunction> functions) + public void setJsonOutputFunctions(Map, ResolvedFunction> functions) { jsonOutputFunctions.putAll(functions); } - public ResolvedFunction getJsonOutputFunction(Expression expression) + public ResolvedFunction getJsonOutputFunction(Node node) + { + return jsonOutputFunctions.get(NodeRef.of(node)); + } + + public void addJsonTableAnalysis(JsonTable jsonTable, JsonTableAnalysis analysis) + { + jsonTableAnalyses.put(NodeRef.of(jsonTable), analysis); + } + + public JsonTableAnalysis getJsonTableAnalysis(JsonTable jsonTable) { - return jsonOutputFunctions.get(NodeRef.of(expression)); + return jsonTableAnalyses.get(NodeRef.of(jsonTable)); } public Map>> getTableColumnReferences() @@ -2333,4 +2351,19 @@ public ConnectorTransactionHandle getTransactionHandle() return transactionHandle; } } + + public record JsonTableAnalysis( + CatalogHandle catalogHandle, + ConnectorTransactionHandle transactionHandle, + Type parametersRowType, + List> orderedOutputColumns) + { + public JsonTableAnalysis + { + requireNonNull(catalogHandle, "catalogHandle is null"); + requireNonNull(transactionHandle, "transactionHandle is null"); + requireNonNull(parametersRowType, "parametersRowType is null"); + requireNonNull(orderedOutputColumns, "orderedOutputColumns is null"); + } + } } diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/ExpressionAnalyzer.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/ExpressionAnalyzer.java index 9cb535691f76..093d171f7a6c 100644 --- a/core/trino-main/src/main/java/io/trino/sql/analyzer/ExpressionAnalyzer.java +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/ExpressionAnalyzer.java @@ -79,6 +79,7 @@ import io.trino.sql.tree.CurrentSchema; import io.trino.sql.tree.CurrentTime; import io.trino.sql.tree.CurrentUser; +import io.trino.sql.tree.DataType; import io.trino.sql.tree.DecimalLiteral; import io.trino.sql.tree.DereferenceExpression; import io.trino.sql.tree.DoubleLiteral; @@ -107,6 +108,7 @@ import io.trino.sql.tree.JsonPathParameter; import io.trino.sql.tree.JsonPathParameter.JsonFormat; import io.trino.sql.tree.JsonQuery; +import io.trino.sql.tree.JsonTable; import io.trino.sql.tree.JsonValue; import io.trino.sql.tree.LambdaArgumentDeclaration; import io.trino.sql.tree.LambdaExpression; @@ -124,6 +126,7 @@ import io.trino.sql.tree.ProcessingMode; import io.trino.sql.tree.QualifiedName; import io.trino.sql.tree.QuantifiedComparisonExpression; +import io.trino.sql.tree.QueryColumn; import io.trino.sql.tree.RangeQuantifier; import io.trino.sql.tree.Row; import io.trino.sql.tree.RowPattern; @@ -140,6 +143,7 @@ import io.trino.sql.tree.TimestampLiteral; import io.trino.sql.tree.Trim; import io.trino.sql.tree.TryExpression; +import io.trino.sql.tree.ValueColumn; import io.trino.sql.tree.VariableDefinition; import io.trino.sql.tree.WhenClause; import io.trino.sql.tree.WindowFrame; @@ -296,7 +300,7 @@ public class ExpressionAnalyzer // Cache from SQL type name to Type; every Type in the cache has a CAST defined from VARCHAR private final Cache varcharCastableTypeCache = buildNonEvictableCache(CacheBuilder.newBuilder().maximumSize(1000)); - private final Map, ResolvedFunction> resolvedFunctions = new LinkedHashMap<>(); + private final Map, ResolvedFunction> resolvedFunctions = new LinkedHashMap<>(); private final Set> subqueries = new LinkedHashSet<>(); private final Set> existsSubqueries = new LinkedHashSet<>(); private final Map, Type> expressionCoercions = new LinkedHashMap<>(); @@ -335,9 +339,9 @@ public class ExpressionAnalyzer private final Set> patternAggregations = new LinkedHashSet<>(); // for JSON functions - private final Map, JsonPathAnalysis> jsonPathAnalyses = new LinkedHashMap<>(); + private final Map, JsonPathAnalysis> jsonPathAnalyses = new LinkedHashMap<>(); private final Map, ResolvedFunction> jsonInputFunctions = new LinkedHashMap<>(); - private final Map, ResolvedFunction> jsonOutputFunctions = new LinkedHashMap<>(); + private final Map, ResolvedFunction> jsonOutputFunctions = new LinkedHashMap<>(); private final Session session; private final Map, Expression> parameters; @@ -399,7 +403,7 @@ private ExpressionAnalyzer( this.getResolvedWindow = requireNonNull(getResolvedWindow, "getResolvedWindow is null"); } - public Map, ResolvedFunction> getResolvedFunctions() + public Map, ResolvedFunction> getResolvedFunctions() { return unmodifiableMap(resolvedFunctions); } @@ -497,6 +501,42 @@ private Type analyze(Expression expression, Scope baseScope, Context context) return visitor.process(expression, new StackableAstVisitor.StackableAstVisitorContext<>(context)); } + private Type analyzeJsonPathInvocation(JsonTable node, Scope scope, CorrelationSupport correlationSupport) + { + Visitor visitor = new Visitor(scope, warningCollector); + List inputTypes = visitor.analyzeJsonPathInvocation("JSON_TABLE", node, node.getJsonPathInvocation(), new StackableAstVisitor.StackableAstVisitorContext<>(Context.notInLambda(scope, correlationSupport))); + return inputTypes.get(2); // parameters row type + } + + private Type analyzeJsonValueExpression(ValueColumn column, JsonPathAnalysis pathAnalysis, Scope scope, CorrelationSupport correlationSupport) + { + Visitor visitor = new Visitor(scope, warningCollector); + List pathInvocationArgumentTypes = ImmutableList.of(JSON_2016, plannerContext.getTypeManager().getType(TypeId.of(JsonPath2016Type.NAME)), JSON_NO_PARAMETERS_ROW_TYPE); + return visitor.analyzeJsonValueExpression( + column, + pathAnalysis, + Optional.of(column.getType()), + pathInvocationArgumentTypes, + column.getEmptyBehavior(), + column.getEmptyDefault(), + column.getErrorBehavior(), + column.getErrorDefault(), + new StackableAstVisitor.StackableAstVisitorContext<>(Context.notInLambda(scope, correlationSupport))); + } + + private Type analyzeJsonQueryExpression(QueryColumn column, Scope scope) + { + Visitor visitor = new Visitor(scope, warningCollector); + List pathInvocationArgumentTypes = ImmutableList.of(JSON_2016, plannerContext.getTypeManager().getType(TypeId.of(JsonPath2016Type.NAME)), JSON_NO_PARAMETERS_ROW_TYPE); + return visitor.analyzeJsonQueryExpression( + column, + column.getWrapperBehavior(), + column.getQuotesBehavior(), + pathInvocationArgumentTypes, + Optional.of(column.getType()), + Optional.of(column.getFormat())); + } + private void analyzeWindow(ResolvedWindow window, Scope scope, Node originalNode, CorrelationSupport correlationSupport) { Visitor visitor = new Visitor(scope, warningCollector); @@ -563,7 +603,7 @@ public Set> getPatternAggregations() return patternAggregations; } - public Map, JsonPathAnalysis> getJsonPathAnalyses() + public Map, JsonPathAnalysis> getJsonPathAnalyses() { return jsonPathAnalyses; } @@ -573,7 +613,7 @@ public Map, ResolvedFunction> getJsonInputFunctions() return jsonInputFunctions; } - public Map, ResolvedFunction> getJsonOutputFunctions() + public Map, ResolvedFunction> getJsonOutputFunctions() { return jsonOutputFunctions; } @@ -2541,15 +2581,38 @@ public Type visitJsonExists(JsonExists node, StackableAstVisitorContext public Type visitJsonValue(JsonValue node, StackableAstVisitorContext context) { List pathInvocationArgumentTypes = analyzeJsonPathInvocation("JSON_VALUE", node, node.getJsonPathInvocation(), context); + Type returnedType = analyzeJsonValueExpression( + node, + jsonPathAnalyses.get(NodeRef.of(node)), + node.getReturnedType(), + pathInvocationArgumentTypes, + node.getEmptyBehavior(), + node.getEmptyDefault(), + Optional.of(node.getErrorBehavior()), + node.getErrorDefault(), + context); + return setExpressionType(node, returnedType); + } + private Type analyzeJsonValueExpression( + Node node, + JsonPathAnalysis pathAnalysis, + Optional declaredReturnedType, + List pathInvocationArgumentTypes, + JsonValue.EmptyOrErrorBehavior emptyBehavior, + Optional declaredEmptyDefault, + Optional errorBehavior, + Optional declaredErrorDefault, + StackableAstVisitorContext context) + { // validate returned type Type returnedType = VARCHAR; // default - if (node.getReturnedType().isPresent()) { + if (declaredReturnedType.isPresent()) { try { - returnedType = plannerContext.getTypeManager().getType(toTypeSignature(node.getReturnedType().get())); + returnedType = plannerContext.getTypeManager().getType(toTypeSignature(declaredReturnedType.get())); } catch (TypeNotFoundException e) { - throw semanticException(TYPE_MISMATCH, node, "Unknown type: %s", node.getReturnedType().get()); + throw semanticException(TYPE_MISMATCH, node, "Unknown type: %s", declaredReturnedType.get()); } } @@ -2559,10 +2622,9 @@ public Type visitJsonValue(JsonValue node, StackableAstVisitorContext c !isDateTimeType(returnedType) || returnedType.equals(INTERVAL_DAY_TIME) || returnedType.equals(INTERVAL_YEAR_MONTH)) { - throw semanticException(TYPE_MISMATCH, node, "Invalid return type of function JSON_VALUE: " + node.getReturnedType().get()); + throw semanticException(TYPE_MISMATCH, node, "Invalid return type of function JSON_VALUE: " + declaredReturnedType.get()); } - JsonPathAnalysis pathAnalysis = jsonPathAnalyses.get(NodeRef.of(node)); Type resultType = pathAnalysis.getType(pathAnalysis.getPath()); if (resultType != null && !resultType.equals(returnedType)) { try { @@ -2574,20 +2636,23 @@ public Type visitJsonValue(JsonValue node, StackableAstVisitorContext c } // validate default values for empty and error behavior - if (node.getEmptyDefault().isPresent()) { - Expression emptyDefault = node.getEmptyDefault().get(); - if (node.getEmptyBehavior() != DEFAULT) { - throw semanticException(INVALID_FUNCTION_ARGUMENT, emptyDefault, "Default value specified for %s ON EMPTY behavior", node.getEmptyBehavior()); + if (declaredEmptyDefault.isPresent()) { + Expression emptyDefault = declaredEmptyDefault.get(); + if (emptyBehavior != DEFAULT) { + throw semanticException(INVALID_FUNCTION_ARGUMENT, emptyDefault, "Default value specified for %s ON EMPTY behavior", emptyBehavior); } Type type = process(emptyDefault, context); // this would normally be done after function resolution, but we know that the default expression is always coerced to the returnedType coerceType(emptyDefault, type, returnedType, "Function JSON_VALUE default ON EMPTY result"); } - if (node.getErrorDefault().isPresent()) { - Expression errorDefault = node.getErrorDefault().get(); - if (node.getErrorBehavior() != DEFAULT) { - throw semanticException(INVALID_FUNCTION_ARGUMENT, errorDefault, "Default value specified for %s ON ERROR behavior", node.getErrorBehavior()); + if (declaredErrorDefault.isPresent()) { + Expression errorDefault = declaredErrorDefault.get(); + if (errorBehavior.isEmpty()) { + throw new IllegalStateException("error default specified without error behavior specified"); + } + if (errorBehavior.orElseThrow() != DEFAULT) { + throw semanticException(INVALID_FUNCTION_ARGUMENT, errorDefault, "Default value specified for %s ON ERROR behavior", errorBehavior.orElseThrow()); } Type type = process(errorDefault, context); // this would normally be done after function resolution, but we know that the default expression is always coerced to the returnedType @@ -2617,21 +2682,32 @@ public Type visitJsonValue(JsonValue node, StackableAstVisitorContext c accessControl.checkCanExecuteFunction(SecurityContext.of(session), JSON_VALUE_FUNCTION_NAME); resolvedFunctions.put(NodeRef.of(node), function); - Type type = function.getSignature().getReturnType(); - return setExpressionType(node, type); + return function.getSignature().getReturnType(); } @Override public Type visitJsonQuery(JsonQuery node, StackableAstVisitorContext context) { List pathInvocationArgumentTypes = analyzeJsonPathInvocation("JSON_QUERY", node, node.getJsonPathInvocation(), context); + Type returnedType = analyzeJsonQueryExpression( + node, + node.getWrapperBehavior(), + node.getQuotesBehavior(), + pathInvocationArgumentTypes, + node.getReturnedType(), + node.getOutputFormat()); + return setExpressionType(node, returnedType); + } - // validate wrapper and quotes behavior - if ((node.getWrapperBehavior() == CONDITIONAL || node.getWrapperBehavior() == UNCONDITIONAL) && node.getQuotesBehavior().isPresent()) { - throw semanticException(INVALID_FUNCTION_ARGUMENT, node, "%s QUOTES behavior specified with WITH %s ARRAY WRAPPER behavior", node.getQuotesBehavior().get(), node.getWrapperBehavior()); - } - + private Type analyzeJsonQueryExpression( + Node node, + JsonQuery.ArrayWrapperBehavior wrapperBehavior, + Optional quotesBehavior, + List pathInvocationArgumentTypes, + Optional declaredReturnedType, + Optional declaredOutputFormat) + { // wrapper behavior, empty behavior and error behavior will be passed as arguments to function // quotes behavior is handled by the corresponding output function List argumentTypes = ImmutableList.builder() @@ -2641,6 +2717,11 @@ public Type visitJsonQuery(JsonQuery node, StackableAstVisitorContext c .add(TINYINT) // error behavior: enum encoded as integer value .build(); + // validate wrapper and quotes behavior + if ((wrapperBehavior == CONDITIONAL || wrapperBehavior == UNCONDITIONAL) && quotesBehavior.isPresent()) { + throw semanticException(INVALID_FUNCTION_ARGUMENT, node, "%s QUOTES behavior specified with WITH %s ARRAY WRAPPER behavior", quotesBehavior.get(), wrapperBehavior); + } + // resolve function ResolvedFunction function; try { @@ -2657,15 +2738,15 @@ public Type visitJsonQuery(JsonQuery node, StackableAstVisitorContext c // analyze returned type and format Type returnedType = VARCHAR; // default - if (node.getReturnedType().isPresent()) { + if (declaredReturnedType.isPresent()) { try { - returnedType = plannerContext.getTypeManager().getType(toTypeSignature(node.getReturnedType().get())); + returnedType = plannerContext.getTypeManager().getType(toTypeSignature(declaredReturnedType.get())); } catch (TypeNotFoundException e) { - throw semanticException(TYPE_MISMATCH, node, "Unknown type: %s", node.getReturnedType().get()); + throw semanticException(TYPE_MISMATCH, node, "Unknown type: %s", declaredReturnedType.get()); } } - JsonFormat outputFormat = node.getOutputFormat().orElse(JsonFormat.JSON); // default + JsonFormat outputFormat = declaredOutputFormat.orElse(JsonFormat.JSON); // default // resolve function to format output ResolvedFunction outputFunction = getOutputFunction(returnedType, outputFormat, node); @@ -2682,13 +2763,15 @@ public Type visitJsonQuery(JsonQuery node, StackableAstVisitorContext c } } - return setExpressionType(node, returnedType); + return returnedType; } - private List analyzeJsonPathInvocation(String functionName, Expression node, JsonPathInvocation jsonPathInvocation, StackableAstVisitorContext context) + private List analyzeJsonPathInvocation(String functionName, Node node, JsonPathInvocation jsonPathInvocation, StackableAstVisitorContext context) { jsonPathInvocation.getPathName().ifPresent(pathName -> { - throw semanticException(INVALID_PATH, pathName, "JSON path name is not allowed in %s function", functionName); + if (!(node instanceof JsonTable)) { + throw semanticException(INVALID_PATH, pathName, "JSON path name is not allowed in %s function", functionName); + } }); // ANALYZE THE CONTEXT ITEM @@ -3458,6 +3541,79 @@ public static ExpressionAnalysis analyzeExpression( analyzer.getWindowFunctions()); } + public static ParametersTypeAndAnalysis analyzeJsonPathInvocation( + JsonTable node, + Session session, + PlannerContext plannerContext, + StatementAnalyzerFactory statementAnalyzerFactory, + AccessControl accessControl, + Scope scope, + Analysis analysis, + WarningCollector warningCollector, + CorrelationSupport correlationSupport) + { + ExpressionAnalyzer analyzer = new ExpressionAnalyzer(plannerContext, accessControl, statementAnalyzerFactory, analysis, session, TypeProvider.empty(), warningCollector); + Type parametersRowType = analyzer.analyzeJsonPathInvocation(node, scope, correlationSupport); + updateAnalysis(analysis, analyzer, session, accessControl); + return new ParametersTypeAndAnalysis( + parametersRowType, + new ExpressionAnalysis( + analyzer.getExpressionTypes(), + analyzer.getExpressionCoercions(), + analyzer.getSubqueryInPredicates(), + analyzer.getSubqueries(), + analyzer.getExistsSubqueries(), + analyzer.getColumnReferences(), + analyzer.getTypeOnlyCoercions(), + analyzer.getQuantifiedComparisons(), + analyzer.getWindowFunctions())); + } + + public record ParametersTypeAndAnalysis(Type parametersRowType, ExpressionAnalysis expressionAnalysis) {} + + public static TypeAndAnalysis analyzeJsonValueExpression( + ValueColumn column, + JsonPathAnalysis pathAnalysis, + Session session, + PlannerContext plannerContext, + StatementAnalyzerFactory statementAnalyzerFactory, + AccessControl accessControl, + Scope scope, + Analysis analysis, + WarningCollector warningCollector, + CorrelationSupport correlationSupport) + { + ExpressionAnalyzer analyzer = new ExpressionAnalyzer(plannerContext, accessControl, statementAnalyzerFactory, analysis, session, TypeProvider.empty(), warningCollector); + Type type = analyzer.analyzeJsonValueExpression(column, pathAnalysis, scope, correlationSupport); + updateAnalysis(analysis, analyzer, session, accessControl); + return new TypeAndAnalysis(type, new ExpressionAnalysis( + analyzer.getExpressionTypes(), + analyzer.getExpressionCoercions(), + analyzer.getSubqueryInPredicates(), + analyzer.getSubqueries(), + analyzer.getExistsSubqueries(), + analyzer.getColumnReferences(), + analyzer.getTypeOnlyCoercions(), + analyzer.getQuantifiedComparisons(), + analyzer.getWindowFunctions())); + } + + public static Type analyzeJsonQueryExpression( + QueryColumn column, + Session session, + PlannerContext plannerContext, + StatementAnalyzerFactory statementAnalyzerFactory, + AccessControl accessControl, + Scope scope, + Analysis analysis, + WarningCollector warningCollector) + { + ExpressionAnalyzer analyzer = new ExpressionAnalyzer(plannerContext, accessControl, statementAnalyzerFactory, analysis, session, TypeProvider.empty(), warningCollector); + Type type = analyzer.analyzeJsonQueryExpression(column, scope); + updateAnalysis(analysis, analyzer, session, accessControl); + return type; + } + public static ExpressionAnalysis analyzeWindow( Session session, PlannerContext plannerContext, @@ -3698,4 +3854,6 @@ public Optional getLabel() return label; } } + + public record TypeAndAnalysis(Type type, ExpressionAnalysis analysis) {} } diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/JsonPathAnalyzer.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/JsonPathAnalyzer.java index c45c523fd9f0..d4d949856978 100644 --- a/core/trino-main/src/main/java/io/trino/sql/analyzer/JsonPathAnalyzer.java +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/JsonPathAnalyzer.java @@ -59,6 +59,7 @@ import io.trino.sql.jsonpath.tree.StartsWithPredicate; import io.trino.sql.jsonpath.tree.TypeMethod; import io.trino.sql.tree.Node; +import io.trino.sql.tree.NodeLocation; import io.trino.sql.tree.QualifiedName; import io.trino.sql.tree.StringLiteral; @@ -110,11 +111,18 @@ public JsonPathAnalysis analyzeJsonPath(StringLiteral path, Map pa Location pathStart = extractLocation(path) .map(location -> new Location(location.getLineNumber(), location.getColumnNumber())) .orElseThrow(() -> new IllegalStateException("missing NodeLocation in path")); - PathNode root = new PathParser(pathStart).parseJsonPath(path.getValue()); + PathNode root = PathParser.withRelativeErrorLocation(pathStart).parseJsonPath(path.getValue()); new Visitor(parameterTypes, path).process(root); return new JsonPathAnalysis((JsonPath) root, types, jsonParameters); } + public JsonPathAnalysis analyzeImplicitJsonPath(String path, NodeLocation location) + { + PathNode root = PathParser.withConstantErrorLocation(new Location(location.getLineNumber(), location.getColumnNumber())).parseJsonPath(path); + new Visitor(ImmutableMap.of(), new StringLiteral(path)).process(root); + return new JsonPathAnalysis((JsonPath) root, types, jsonParameters); + } + /** * This visitor determines and validates output types of PathNodes, whenever they can be deduced and represented as SQL types. * In some cases, the type of a PathNode can be determined without context. E.g., the `double()` method always returns DOUBLE. diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java index 594e9c74f12b..5bc45da60dd6 100644 --- a/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java @@ -23,11 +23,13 @@ import com.google.common.collect.Iterables; import com.google.common.collect.ListMultimap; import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; import com.google.common.collect.Streams; import com.google.common.math.IntMath; import io.airlift.slice.Slice; import io.trino.Session; import io.trino.SystemSessionProperties; +import io.trino.connector.system.GlobalSystemConnector; import io.trino.execution.Column; import io.trino.execution.warnings.WarningCollector; import io.trino.metadata.AnalyzePropertyManager; @@ -105,6 +107,7 @@ import io.trino.sql.PlannerContext; import io.trino.sql.SqlPath; import io.trino.sql.analyzer.Analysis.GroupingSetAnalysis; +import io.trino.sql.analyzer.Analysis.JsonTableAnalysis; import io.trino.sql.analyzer.Analysis.MergeAnalysis; import io.trino.sql.analyzer.Analysis.ResolvedWindow; import io.trino.sql.analyzer.Analysis.SelectExpression; @@ -112,6 +115,9 @@ import io.trino.sql.analyzer.Analysis.TableArgumentAnalysis; import io.trino.sql.analyzer.Analysis.TableFunctionInvocationAnalysis; import io.trino.sql.analyzer.Analysis.UnnestAnalysis; +import io.trino.sql.analyzer.ExpressionAnalyzer.ParametersTypeAndAnalysis; +import io.trino.sql.analyzer.ExpressionAnalyzer.TypeAndAnalysis; +import io.trino.sql.analyzer.JsonPathAnalyzer.JsonPathAnalysis; import io.trino.sql.analyzer.PatternRecognitionAnalyzer.PatternRecognitionAnalysis; import io.trino.sql.analyzer.Scope.AsteriskedIdentifierChainBasis; import io.trino.sql.parser.ParsingException; @@ -172,7 +178,11 @@ import io.trino.sql.tree.JoinCriteria; import io.trino.sql.tree.JoinOn; import io.trino.sql.tree.JoinUsing; +import io.trino.sql.tree.JsonPathInvocation; +import io.trino.sql.tree.JsonPathParameter; import io.trino.sql.tree.JsonTable; +import io.trino.sql.tree.JsonTableColumnDefinition; +import io.trino.sql.tree.JsonTableSpecificPlan; import io.trino.sql.tree.Lateral; import io.trino.sql.tree.Limit; import io.trino.sql.tree.LongLiteral; @@ -183,16 +193,23 @@ import io.trino.sql.tree.MergeInsert; import io.trino.sql.tree.MergeUpdate; import io.trino.sql.tree.NaturalJoin; +import io.trino.sql.tree.NestedColumns; import io.trino.sql.tree.Node; +import io.trino.sql.tree.NodeLocation; import io.trino.sql.tree.NodeRef; import io.trino.sql.tree.Offset; import io.trino.sql.tree.OrderBy; +import io.trino.sql.tree.OrdinalityColumn; import io.trino.sql.tree.Parameter; import io.trino.sql.tree.PatternRecognitionRelation; +import io.trino.sql.tree.PlanLeaf; +import io.trino.sql.tree.PlanParentChild; +import io.trino.sql.tree.PlanSiblings; import io.trino.sql.tree.Prepare; import io.trino.sql.tree.Property; import io.trino.sql.tree.QualifiedName; import io.trino.sql.tree.Query; +import io.trino.sql.tree.QueryColumn; import io.trino.sql.tree.QueryPeriod; import io.trino.sql.tree.QuerySpecification; import io.trino.sql.tree.RefreshMaterializedView; @@ -223,6 +240,7 @@ import io.trino.sql.tree.SortItem; import io.trino.sql.tree.StartTransaction; import io.trino.sql.tree.Statement; +import io.trino.sql.tree.StringLiteral; import io.trino.sql.tree.SubqueryExpression; import io.trino.sql.tree.SubscriptExpression; import io.trino.sql.tree.Table; @@ -238,6 +256,7 @@ import io.trino.sql.tree.Update; import io.trino.sql.tree.UpdateAssignment; import io.trino.sql.tree.Use; +import io.trino.sql.tree.ValueColumn; import io.trino.sql.tree.Values; import io.trino.sql.tree.VariableDefinition; import io.trino.sql.tree.Window; @@ -290,6 +309,7 @@ import static io.trino.spi.StandardErrorCode.COLUMN_NOT_FOUND; import static io.trino.spi.StandardErrorCode.COLUMN_TYPE_UNKNOWN; import static io.trino.spi.StandardErrorCode.DUPLICATE_COLUMN_NAME; +import static io.trino.spi.StandardErrorCode.DUPLICATE_COLUMN_OR_PATH_NAME; import static io.trino.spi.StandardErrorCode.DUPLICATE_NAMED_QUERY; import static io.trino.spi.StandardErrorCode.DUPLICATE_PROPERTY; import static io.trino.spi.StandardErrorCode.DUPLICATE_RANGE_VARIABLE; @@ -308,6 +328,7 @@ import static io.trino.spi.StandardErrorCode.INVALID_LIMIT_CLAUSE; import static io.trino.spi.StandardErrorCode.INVALID_ORDER_BY; import static io.trino.spi.StandardErrorCode.INVALID_PARTITION_BY; +import static io.trino.spi.StandardErrorCode.INVALID_PLAN; import static io.trino.spi.StandardErrorCode.INVALID_RECURSIVE_REFERENCE; import static io.trino.spi.StandardErrorCode.INVALID_ROW_FILTER; import static io.trino.spi.StandardErrorCode.INVALID_TABLE_FUNCTION_INVOCATION; @@ -320,6 +341,7 @@ import static io.trino.spi.StandardErrorCode.MISSING_COLUMN_NAME; import static io.trino.spi.StandardErrorCode.MISSING_GROUP_BY; import static io.trino.spi.StandardErrorCode.MISSING_ORDER_BY; +import static io.trino.spi.StandardErrorCode.MISSING_PATH_NAME; import static io.trino.spi.StandardErrorCode.MISSING_RETURN_TYPE; import static io.trino.spi.StandardErrorCode.NESTED_RECURSIVE; import static io.trino.spi.StandardErrorCode.NESTED_ROW_PATTERN_RECOGNITION; @@ -357,6 +379,8 @@ import static io.trino.sql.analyzer.AggregationAnalyzer.verifySourceAggregations; import static io.trino.sql.analyzer.Analyzer.verifyNoAggregateWindowOrGroupingFunctions; import static io.trino.sql.analyzer.CanonicalizationAware.canonicalizationAwareKey; +import static io.trino.sql.analyzer.ExpressionAnalyzer.analyzeJsonQueryExpression; +import static io.trino.sql.analyzer.ExpressionAnalyzer.analyzeJsonValueExpression; import static io.trino.sql.analyzer.ExpressionAnalyzer.createConstantAnalyzer; import static io.trino.sql.analyzer.ExpressionTreeUtils.asQualifiedName; import static io.trino.sql.analyzer.ExpressionTreeUtils.extractAggregateFunctions; @@ -3180,6 +3204,17 @@ protected Scope visitJoin(Join node, Optional scope) } } } + else if (isJsonTable(node.getRight())) { + if (criteria != null) { + if (!(criteria instanceof JoinOn) || !((JoinOn) criteria).getExpression().equals(TRUE_LITERAL)) { + throw semanticException( + NOT_SUPPORTED, + criteria instanceof JoinOn ? ((JoinOn) criteria).getExpression() : node, + "%s JOIN involving JSON_TABLE is only supported with condition ON TRUE", + node.getType().name()); + } + } + } else if (node.getType() == FULL) { if (!(criteria instanceof JoinOn) || !((JoinOn) criteria).getExpression().equals(TRUE_LITERAL)) { throw semanticException( @@ -3696,7 +3731,7 @@ private boolean isLateralRelation(Relation node) if (node instanceof AliasedRelation) { return isLateralRelation(((AliasedRelation) node).getRelation()); } - return node instanceof Unnest || node instanceof Lateral; + return node instanceof Unnest || node instanceof Lateral || node instanceof JsonTable; } private boolean isUnnestRelation(Relation node) @@ -3707,6 +3742,14 @@ private boolean isUnnestRelation(Relation node) return node instanceof Unnest; } + private boolean isJsonTable(Relation node) + { + if (node instanceof AliasedRelation) { + return isJsonTable(((AliasedRelation) node).getRelation()); + } + return node instanceof JsonTable; + } + @Override protected Scope visitValues(Values node, Optional scope) { @@ -3782,9 +3825,255 @@ else if (actualType instanceof RowType) { } @Override - protected Scope visitJsonTable(JsonTable node, Optional context) + protected Scope visitJsonTable(JsonTable node, Optional scope) + { + Scope enclosingScope = createScope(scope); + + // analyze the context item, the root JSON path, and the path parameters + Type parametersRowType = analyzeJsonPathInvocation(node, enclosingScope); + + // json_table is implemented as a table function provided by the global catalog. + CatalogHandle catalogHandle = getRequiredCatalogHandle(metadata, session, node, GlobalSystemConnector.NAME); + ConnectorTransactionHandle transactionHandle = transactionManager.getConnectorTransaction(session.getRequiredTransactionId(), catalogHandle); + + // all column and path names must be unique + Set uniqueNames = new HashSet<>(); + JsonPathInvocation rootPath = node.getJsonPathInvocation(); + rootPath.getPathName().ifPresent(name -> uniqueNames.add(name.getCanonicalValue())); + + ImmutableList.Builder outputFields = ImmutableList.builder(); + ImmutableList.Builder> orderedOutputColumns = ImmutableList.builder(); + analyzeJsonTableColumns(node.getColumns(), uniqueNames, outputFields, orderedOutputColumns, enclosingScope, node); + + analysis.addJsonTableAnalysis(node, new JsonTableAnalysis(catalogHandle, transactionHandle, parametersRowType, orderedOutputColumns.build())); + + node.getPlan().ifPresent(plan -> { + if (plan instanceof JsonTableSpecificPlan specificPlan) { + validateJsonTableSpecificPlan(rootPath, specificPlan, node.getColumns()); + } + else { + // if PLAN DEFAULT is specified, all nested paths should be named + checkAllNestedPathsNamed(node.getColumns()); + } + }); + + // TODO should we do an access control check if we can execute json_table? IMO no, as this is a language construct like UNNEST or LATERAL + return createAndAssignScope(node, scope, outputFields.build()); + } + + private Type analyzeJsonPathInvocation(JsonTable node, Scope scope) + { + verifyNoAggregateWindowOrGroupingFunctions(session, metadata, node.getJsonPathInvocation().getInputExpression(), "JSON_TABLE input expression"); + node.getJsonPathInvocation().getPathParameters().stream() + .map(JsonPathParameter::getParameter) + .forEach(parameter -> verifyNoAggregateWindowOrGroupingFunctions(session, metadata, parameter, "JSON_TABLE path parameter")); + + ParametersTypeAndAnalysis parametersTypeAndAnalysis = ExpressionAnalyzer.analyzeJsonPathInvocation( + node, + session, + plannerContext, + statementAnalyzerFactory, + accessControl, + scope, + analysis, + WarningCollector.NOOP, + correlationSupport); + // context item and passed path parameters can contain subqueries - the subqueries are recorded under the enclosing JsonTable node + analysis.recordSubqueries(node, parametersTypeAndAnalysis.expressionAnalysis()); + return parametersTypeAndAnalysis.parametersRowType(); + } + + private void analyzeJsonTableColumns( + List columns, + Set uniqueNames, + ImmutableList.Builder outputFields, + ImmutableList.Builder> orderedOutputColumns, + Scope enclosingScope, + JsonTable jsonTable) + { + for (JsonTableColumnDefinition column : columns) { + if (column instanceof OrdinalityColumn ordinalityColumn) { + String name = ordinalityColumn.getName().getCanonicalValue(); + if (!uniqueNames.add(name)) { + throw semanticException(DUPLICATE_COLUMN_OR_PATH_NAME, ordinalityColumn.getName(), "All column and path names in JSON_TABLE invocation must be unique"); + } + outputFields.add(Field.newUnqualified(name, BIGINT)); + orderedOutputColumns.add(NodeRef.of(ordinalityColumn)); + } + else if (column instanceof ValueColumn valueColumn) { + String name = valueColumn.getName().getCanonicalValue(); + if (!uniqueNames.add(name)) { + throw semanticException(DUPLICATE_COLUMN_OR_PATH_NAME, valueColumn.getName(), "All column and path names in JSON_TABLE invocation must be unique"); + } + valueColumn.getEmptyDefault().ifPresent(expression -> verifyNoAggregateWindowOrGroupingFunctions(session, metadata, expression, "default expression for JSON_TABLE column")); + valueColumn.getErrorDefault().ifPresent(expression -> verifyNoAggregateWindowOrGroupingFunctions(session, metadata, expression, "default expression for JSON_TABLE column")); + JsonPathAnalysis pathAnalysis = valueColumn.getJsonPath() + .map(this::analyzeJsonPath) + .orElseGet(() -> analyzeImplicitJsonPath(getImplicitJsonPath(name), valueColumn.getLocation())); + analysis.setJsonPathAnalysis(valueColumn, pathAnalysis); + TypeAndAnalysis typeAndAnalysis = analyzeJsonValueExpression( + valueColumn, + pathAnalysis, + session, + plannerContext, + statementAnalyzerFactory, + accessControl, + enclosingScope, + analysis, + warningCollector, + correlationSupport); + // default values can contain subqueries - the subqueries are recorded under the enclosing JsonTable node + analysis.recordSubqueries(jsonTable, typeAndAnalysis.analysis()); + outputFields.add(Field.newUnqualified(name, typeAndAnalysis.type())); + orderedOutputColumns.add(NodeRef.of(valueColumn)); + } + else if (column instanceof QueryColumn queryColumn) { + String name = queryColumn.getName().getCanonicalValue(); + if (!uniqueNames.add(name)) { + throw semanticException(DUPLICATE_COLUMN_OR_PATH_NAME, queryColumn.getName(), "All column and path names in JSON_TABLE invocation must be unique"); + } + JsonPathAnalysis pathAnalysis = queryColumn.getJsonPath() + .map(this::analyzeJsonPath) + .orElseGet(() -> analyzeImplicitJsonPath(getImplicitJsonPath(name), queryColumn.getLocation())); + analysis.setJsonPathAnalysis(queryColumn, pathAnalysis); + Type type = analyzeJsonQueryExpression(queryColumn, session, plannerContext, statementAnalyzerFactory, accessControl, enclosingScope, analysis, warningCollector); + outputFields.add(Field.newUnqualified(name, type)); + orderedOutputColumns.add(NodeRef.of(queryColumn)); + } + else if (column instanceof NestedColumns nestedColumns) { + nestedColumns.getPathName().ifPresent(name -> { + if (!uniqueNames.add(name.getCanonicalValue())) { + throw semanticException(DUPLICATE_COLUMN_OR_PATH_NAME, name, "All column and path names in JSON_TABLE invocation must be unique"); + } + }); + JsonPathAnalysis pathAnalysis = analyzeJsonPath(nestedColumns.getJsonPath()); + analysis.setJsonPathAnalysis(nestedColumns, pathAnalysis); + analyzeJsonTableColumns(nestedColumns.getColumns(), uniqueNames, outputFields, orderedOutputColumns, enclosingScope, jsonTable); + } + else { + throw new IllegalArgumentException("unexpected type of JSON_TABLE column: " + column.getClass().getSimpleName()); + } + } + } + + private static String getImplicitJsonPath(String name) + { + // TODO the spec misses the path mode. I put 'lax', but it should be confirmed, as the path mode is meaningful for the semantics of the implicit path. + return "lax $.\"" + name.replace("\"", "\"\"") + '"'; + } + + private JsonPathAnalysis analyzeJsonPath(StringLiteral path) + { + return new JsonPathAnalyzer( + plannerContext.getMetadata(), + session, + createConstantAnalyzer(plannerContext, accessControl, session, analysis.getParameters(), WarningCollector.NOOP, analysis.isDescribe())) + .analyzeJsonPath(path, ImmutableMap.of()); + } + + private JsonPathAnalysis analyzeImplicitJsonPath(String path, Optional columnLocation) + { + return new JsonPathAnalyzer( + plannerContext.getMetadata(), + session, + createConstantAnalyzer(plannerContext, accessControl, session, analysis.getParameters(), WarningCollector.NOOP, analysis.isDescribe())) + .analyzeImplicitJsonPath(path, columnLocation.orElseThrow(() -> new IllegalStateException("missing NodeLocation for JSON_TABLE column"))); + } + + private void validateJsonTableSpecificPlan(JsonPathInvocation rootPath, JsonTableSpecificPlan rootPlan, List rootColumns) + { + String rootPathName = rootPath.getPathName() + .orElseThrow(() -> semanticException(MISSING_PATH_NAME, rootPath, "All JSON paths must be named when specific plan is given")) + .getCanonicalValue(); + String rootPlanName; + if (rootPlan instanceof PlanLeaf planLeaf) { + rootPlanName = planLeaf.getName().getCanonicalValue(); + } + else if (rootPlan instanceof PlanParentChild planParentChild) { + rootPlanName = planParentChild.getParent().getName().getCanonicalValue(); + } + else { + throw semanticException(INVALID_PLAN, rootPlan, "JSON_TABLE plan must either be a single path name or it must be rooted in parent-child relationship (OUTER or INNER)"); + } + validateJsonTablePlan(ImmutableMap.of(rootPathName, rootColumns), ImmutableMap.of(rootPlanName, rootPlan), rootPlan); + } + + private void validateJsonTablePlan(Map> actualNodes, Map planNodes, JsonTableSpecificPlan rootPlan) + { + Set unhandledActualNodes = Sets.difference(actualNodes.keySet(), planNodes.keySet()); + if (!unhandledActualNodes.isEmpty()) { + throw semanticException(INVALID_PLAN, rootPlan, "JSON_TABLE plan should contain all JSON paths available at each level of nesting. Paths not included: " + String.join(", ", unhandledActualNodes)); + } + Set irrelevantPlanChildren = Sets.difference(planNodes.keySet(), actualNodes.keySet()); + if (!irrelevantPlanChildren.isEmpty()) { + throw semanticException(INVALID_PLAN, rootPlan, "JSON_TABLE plan includes unavailable JSON path names: " + String.join(", ", irrelevantPlanChildren)); + } + + // recurse into child nodes + actualNodes.forEach((name, columns) -> { + JsonTableSpecificPlan plan = planNodes.get(name); + + Map> actualChildren = columns.stream() + .filter(NestedColumns.class::isInstance) + .map(NestedColumns.class::cast) + .collect(toImmutableMap( + child -> child.getPathName() + .orElseThrow(() -> semanticException(MISSING_PATH_NAME, child.getJsonPath(), "All JSON paths must be named when specific plan is given")) + .getCanonicalValue(), + NestedColumns::getColumns)); + + Map planChildren; + if (plan instanceof PlanLeaf) { + planChildren = ImmutableMap.of(); + } + else if (plan instanceof PlanParentChild planParentChild) { + planChildren = new HashMap<>(); + getPlanSiblings(planParentChild.getChild(), planChildren); + } + else { + throw new IllegalStateException("unexpected JSON_TABLE plan node: " + plan.getClass().getSimpleName()); + } + + validateJsonTablePlan(actualChildren, planChildren, rootPlan); + }); + } + + private void getPlanSiblings(JsonTableSpecificPlan plan, Map map) + { + if (plan instanceof PlanLeaf planLeaf) { + if (map.put(planLeaf.getName().getCanonicalValue(), planLeaf) != null) { + throw semanticException(INVALID_PLAN, planLeaf, "Duplicate reference to JSON path name in sibling plan: " + planLeaf.getName().getCanonicalValue()); + } + } + else if (plan instanceof PlanParentChild planParentChild) { + if (map.put(planParentChild.getParent().getName().getCanonicalValue(), planParentChild) != null) { + throw semanticException(INVALID_PLAN, planParentChild.getParent(), "Duplicate reference to JSON path name in sibling plan: " + planParentChild.getParent().getName().getCanonicalValue()); + } + } + else if (plan instanceof PlanSiblings planSiblings) { + for (JsonTableSpecificPlan sibling : planSiblings.getSiblings()) { + getPlanSiblings(sibling, map); + } + } + } + + // Per SQL standard ISO/IEC STANDARD 9075-2, p. 453, g), i), and p. 821, 2), b), when PLAN DEFAULT is specified, all nested paths must be named, but the root path does not have to be named. + private void checkAllNestedPathsNamed(List columns) { - throw semanticException(NOT_SUPPORTED, node, "JSON_TABLE is not yet supported"); + List nestedColumns = columns.stream() + .filter(NestedColumns.class::isInstance) + .map(NestedColumns.class::cast) + .collect(toImmutableList()); + + nestedColumns.stream() + .forEach(definition -> { + if (definition.getPathName().isEmpty()) { + throw semanticException(MISSING_PATH_NAME, definition.getJsonPath(), "All nested JSON paths must be named when default plan is given"); + } + }); + + nestedColumns.stream() + .forEach(definition -> checkAllNestedPathsNamed(definition.getColumns())); } private void analyzeWindowDefinitions(QuerySpecification node, Scope scope) diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/RelationPlanner.java b/core/trino-main/src/main/java/io/trino/sql/planner/RelationPlanner.java index ae1527599426..63223834836c 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/RelationPlanner.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/RelationPlanner.java @@ -72,6 +72,7 @@ import io.trino.sql.tree.Join; import io.trino.sql.tree.JoinCriteria; import io.trino.sql.tree.JoinUsing; +import io.trino.sql.tree.JsonTable; import io.trino.sql.tree.LambdaArgumentDeclaration; import io.trino.sql.tree.Lateral; import io.trino.sql.tree.MeasureDefinition; @@ -1206,6 +1207,12 @@ private PlanBuilder planSingleEmptyRow(Optional parent) return new PlanBuilder(translations, values); } + @Override + protected RelationPlan visitJsonTable(JsonTable node, Void context) + { + throw semanticException(NOT_SUPPORTED, node, "JSON_TABLE is not yet supported"); + } + @Override protected RelationPlan visitUnion(Union node, Void context) { diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/ResolvedFunctionCallRewriter.java b/core/trino-main/src/main/java/io/trino/sql/planner/ResolvedFunctionCallRewriter.java index 2a5e457e611a..8b2c6c985106 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/ResolvedFunctionCallRewriter.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/ResolvedFunctionCallRewriter.java @@ -18,6 +18,7 @@ import io.trino.sql.tree.ExpressionRewriter; import io.trino.sql.tree.ExpressionTreeRewriter; import io.trino.sql.tree.FunctionCall; +import io.trino.sql.tree.Node; import io.trino.sql.tree.NodeRef; import java.util.Map; @@ -29,7 +30,7 @@ public final class ResolvedFunctionCallRewriter { private ResolvedFunctionCallRewriter() {} - public static Expression rewriteResolvedFunctions(Expression expression, Map, ResolvedFunction> resolvedFunctions) + public static Expression rewriteResolvedFunctions(Expression expression, Map, ResolvedFunction> resolvedFunctions) { return ExpressionTreeRewriter.rewriteWith(new Visitor(resolvedFunctions), expression); } @@ -37,9 +38,9 @@ public static Expression rewriteResolvedFunctions(Expression expression, Map { - private final Map, ResolvedFunction> resolvedFunctions; + private final Map, ResolvedFunction> resolvedFunctions; - public Visitor(Map, ResolvedFunction> resolvedFunctions) + public Visitor(Map, ResolvedFunction> resolvedFunctions) { this.resolvedFunctions = requireNonNull(resolvedFunctions, "resolvedFunctions is null"); } diff --git a/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java b/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java index fb8e0110f698..5e73beb3a7f5 100644 --- a/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java +++ b/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java @@ -114,6 +114,7 @@ import static io.trino.spi.StandardErrorCode.COLUMN_NOT_FOUND; import static io.trino.spi.StandardErrorCode.COLUMN_TYPE_UNKNOWN; import static io.trino.spi.StandardErrorCode.DUPLICATE_COLUMN_NAME; +import static io.trino.spi.StandardErrorCode.DUPLICATE_COLUMN_OR_PATH_NAME; import static io.trino.spi.StandardErrorCode.DUPLICATE_NAMED_QUERY; import static io.trino.spi.StandardErrorCode.DUPLICATE_PARAMETER_NAME; import static io.trino.spi.StandardErrorCode.DUPLICATE_PROPERTY; @@ -139,6 +140,7 @@ import static io.trino.spi.StandardErrorCode.INVALID_PARTITION_BY; import static io.trino.spi.StandardErrorCode.INVALID_PATH; import static io.trino.spi.StandardErrorCode.INVALID_PATTERN_RECOGNITION_FUNCTION; +import static io.trino.spi.StandardErrorCode.INVALID_PLAN; import static io.trino.spi.StandardErrorCode.INVALID_PROCESSING_MODE; import static io.trino.spi.StandardErrorCode.INVALID_RANGE; import static io.trino.spi.StandardErrorCode.INVALID_RECURSIVE_REFERENCE; @@ -156,6 +158,7 @@ import static io.trino.spi.StandardErrorCode.MISSING_GROUP_BY; import static io.trino.spi.StandardErrorCode.MISSING_ORDER_BY; import static io.trino.spi.StandardErrorCode.MISSING_OVER; +import static io.trino.spi.StandardErrorCode.MISSING_PATH_NAME; import static io.trino.spi.StandardErrorCode.MISSING_ROW_PATTERN; import static io.trino.spi.StandardErrorCode.MISSING_SCHEMA_NAME; import static io.trino.spi.StandardErrorCode.MISSING_VARIABLE_DEFINITIONS; @@ -6674,11 +6677,541 @@ public void testTableFunctionRequiredColumns() } @Test - public void testJsonTable() + public void testJsonTableColumnTypes() { - assertFails("SELECT * FROM JSON_TABLE('[1, 2, 3]', 'lax $[2]' COLUMNS(o FOR ORDINALITY))") + // ordinality column + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS( + o FOR ORDINALITY)) + """); + + // regular column + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' + COLUMNS( + id BIGINT + PATH 'lax $[1]' + DEFAULT 0 ON EMPTY + ERROR ON ERROR)) + """); + + // formatted column + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' + COLUMNS( + id VARBINARY + FORMAT JSON ENCODING UTF16 + PATH 'lax $[1]' + WITHOUT WRAPPER + OMIT QUOTES + EMPTY ARRAY ON EMPTY + NULL ON ERROR)) + """); + + // nested columns + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' + COLUMNS( + NESTED PATH 'lax $[*]' AS nested_path COLUMNS ( + o FOR ORDINALITY, + id BIGINT PATH 'lax $[1]'))) + """); + } + + @Test + public void testJsonTableColumnAndPathNameUniqueness() + { + // root path is named + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' AS root_path + COLUMNS( + o FOR ORDINALITY)) + """); + + // nested path is named + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' + COLUMNS( + NESTED PATH 'lax $[*]' AS nested_path COLUMNS ( + o FOR ORDINALITY))) + """); + + // root and nested paths are named + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + NESTED PATH 'lax $[*]' AS nested_path COLUMNS ( + o FOR ORDINALITY))) + """); + + // duplicate path name + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS some_path + COLUMNS( + NESTED PATH 'lax $[*]' AS some_path COLUMNS ( + o FOR ORDINALITY))) + """) + .hasErrorCode(DUPLICATE_COLUMN_OR_PATH_NAME) + .hasMessage("line 6:35: All column and path names in JSON_TABLE invocation must be unique"); + + // duplicate column name + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS( + id FOR ORDINALITY, + id BIGINT)) + """) + .hasErrorCode(DUPLICATE_COLUMN_OR_PATH_NAME) + .hasMessage("line 7:9: All column and path names in JSON_TABLE invocation must be unique"); + + // column and path names are the same + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' AS some_name + COLUMNS( + some_name FOR ORDINALITY)) + """) + .hasErrorCode(DUPLICATE_COLUMN_OR_PATH_NAME) + .hasMessage("line 6:9: All column and path names in JSON_TABLE invocation must be unique"); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' + COLUMNS( + NESTED PATH 'lax $[*]' AS some_name COLUMNS ( + some_name FOR ORDINALITY))) + """) + .hasErrorCode(DUPLICATE_COLUMN_OR_PATH_NAME) + .hasMessage("line 7:13: All column and path names in JSON_TABLE invocation must be unique"); + + // duplicate name is deeply nested + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS( + NESTED PATH 'lax $[*]' AS some_name COLUMNS ( + NESTED PATH 'lax $' AS another_name COLUMNS ( + NESTED PATH 'lax $' AS yet_another_name COLUMNS ( + some_name FOR ORDINALITY))))) + """) + .hasErrorCode(DUPLICATE_COLUMN_OR_PATH_NAME) + .hasMessage("line 9:21: All column and path names in JSON_TABLE invocation must be unique"); + } + + @Test + public void testJsonTableColumnAndPathNameIdentifierSemantics() + { + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' AS some_name + COLUMNS( + Some_Name FOR ORDINALITY)) + """) + .hasErrorCode(DUPLICATE_COLUMN_OR_PATH_NAME) + .hasMessage("line 6:9: All column and path names in JSON_TABLE invocation must be unique"); + + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' AS some_name + COLUMNS( + "some_name" FOR ORDINALITY)) + """); + } + + @Test + public void testJsonTableOutputColumns() + { + analyze(""" + SELECT a, b, c, d, e + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' + COLUMNS( + a FOR ORDINALITY, + b BIGINT, + c VARBINARY FORMAT JSON ENCODING UTF16, + NESTED PATH 'lax $[*]' COLUMNS ( + d FOR ORDINALITY, + e BIGINT))) + """); + } + + @Test + public void testImplicitJsonPath() + { + // column name: Ab + // canonical name: AB + // implicit path: lax $."AB" + // resolved member accessor: $.AB + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS(Ab BIGINT)) + """); + + // column name: Ab + // canonical name: Ab + // implicit path: lax $."Ab" + // resolved member accessor: $.Ab + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS("Ab" BIGINT)) + """); + + // column name: ? + // canonical name: ? + // implicit path: lax $."?" + // resolved member accessor: $.? + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS("?" BIGINT)) + """); + + // column name: " + // canonical name: " + // implicit path: lax $."""" + // resolved member accessor $." + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS("\"\"" BIGINT)) + """); + } + + @Test + public void testJsonTableSpecificPlan() + { + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS(id BIGINT) + PLAN (root_path)) + """) + .hasErrorCode(MISSING_PATH_NAME) + .hasMessage("line 3:5: All JSON paths must be named when specific plan is given"); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' AS root_path + COLUMNS(id BIGINT) + PLAN (root_path UNION another_path)) + """) + .hasErrorCode(INVALID_PLAN) + .hasMessage("line 6:11: JSON_TABLE plan must either be a single path name or it must be rooted in parent-child relationship (OUTER or INNER)"); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS(id BIGINT) + PLAN (another_path)) + """) + .hasErrorCode(INVALID_PLAN) + .hasMessage("line 6:11: JSON_TABLE plan should contain all JSON paths available at each level of nesting. Paths not included: ROOT_PATH"); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + NESTED PATH 'lax $' COLUMNS(id BIGINT)) + PLAN (root_path OUTER another_path)) + """) + .hasErrorCode(MISSING_PATH_NAME) + .hasMessage("line 6:21: All JSON paths must be named when specific plan is given"); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + NESTED PATH 'lax $' AS nested_path_1 COLUMNS(id_1 BIGINT), + NESTED PATH 'lax $' AS nested_path_2 COLUMNS(id_2 BIGINT)) + PLAN (root_path OUTER (nested_path_1 CROSS another_path))) + """) + .hasErrorCode(INVALID_PLAN) + .hasMessage("line 8:11: JSON_TABLE plan should contain all JSON paths available at each level of nesting. Paths not included: NESTED_PATH_2"); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + NESTED PATH 'lax $' AS nested_path_1 COLUMNS(id_1 BIGINT), + NESTED PATH 'lax $' AS nested_path_2 COLUMNS(id_2 BIGINT)) + PLAN (root_path OUTER (nested_path_1 CROSS another_path CROSS nested_path_2))) + """) + .hasErrorCode(INVALID_PLAN) + .hasMessage("line 8:11: JSON_TABLE plan includes unavailable JSON path names: ANOTHER_PATH"); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + NESTED PATH 'lax $' AS nested_path_1 COLUMNS(id_1 BIGINT), + NESTED PATH 'lax $' AS nested_path_2 COLUMNS( + id_2 BIGINT, + NESTED PATH 'lax $' AS nested_path_3 COLUMNS(id_3 BIGINT))) + PLAN (root_path OUTER (nested_path_1 CROSS (nested_path_2 UNION nested_path_3)))) + """) + .hasErrorCode(INVALID_PLAN) + .hasMessage("line 10:11: JSON_TABLE plan includes unavailable JSON path names: NESTED_PATH_3"); // nested_path_3 is on another nesting level + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + NESTED PATH 'lax $' AS nested_path_1 COLUMNS(id_1 BIGINT), + NESTED PATH 'lax $' AS nested_path_2 COLUMNS(id_2 BIGINT)) + PLAN (root_path OUTER (nested_path_1 CROSS (nested_path_2 UNION nested_path_1)))) + """) + .hasErrorCode(INVALID_PLAN) + .hasMessage("line 8:69: Duplicate reference to JSON path name in sibling plan: NESTED_PATH_1"); + + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + NESTED PATH 'lax $' AS nested_path_1 COLUMNS(id_1 BIGINT), + NESTED PATH 'lax $' AS nested_path_2 COLUMNS( + id_2 BIGINT, + NESTED PATH 'lax $' AS nested_path_3 COLUMNS(id_3 BIGINT))) + PLAN (root_path OUTER (nested_path_1 CROSS (nested_path_2 INNER nested_path_3)))) + """); + } + + @Test + public void testJsonTableDefaultPlan() + { + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS(id BIGINT) + PLAN DEFAULT(CROSS, INNER)) + """); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + NESTED PATH 'lax $' COLUMNS(id BIGINT)) + PLAN DEFAULT(OUTER, UNION)) + """) + .hasErrorCode(MISSING_PATH_NAME) + .hasMessage("line 6:21: All nested JSON paths must be named when default plan is given"); + } + + @Test + public void tstJsonTableInJoin() + { + analyze(""" + SELECT * + FROM t1, t2, JSON_TABLE('[1, 2, 3]', 'lax $[2]' COLUMNS(o FOR ORDINALITY)) + """); + + // join condition + analyze(""" + SELECT * + FROM t1 + LEFT JOIN + JSON_TABLE('[1, 2, 3]', 'lax $[2]' COLUMNS(o FOR ORDINALITY)) + ON TRUE + """); + + assertFails(""" + SELECT * + FROM t1 + RIGHT JOIN + JSON_TABLE('[1, 2, 3]', 'lax $[2]' COLUMNS(o FOR ORDINALITY)) t + ON t.o > t1.a + """) .hasErrorCode(NOT_SUPPORTED) - .hasMessage("line 1:15: JSON_TABLE is not yet supported"); + .hasMessage("line 5:12: RIGHT JOIN involving JSON_TABLE is only supported with condition ON TRUE"); + + // correlation in context item + analyze(""" + SELECT * + FROM t6 + LEFT JOIN + JSON_TABLE(b, 'lax $[2]' COLUMNS(o FOR ORDINALITY)) + ON TRUE + """); + + // correlation in default value + analyze(""" + SELECT * + FROM t6 + LEFT JOIN + JSON_TABLE('[1, 2, 3]', 'lax $[2]' COLUMNS(x BIGINT DEFAULT a ON EMPTY)) + ON TRUE + """); + + // correlation in path parameter + analyze(""" + SELECT * + FROM t6 + LEFT JOIN + JSON_TABLE('[1, 2, 3]', 'lax $[2]' PASSING a AS parameter_name COLUMNS(o FOR ORDINALITY)) + ON TRUE + """); + + // invalid correlation in right join + assertFails(""" + SELECT * + FROM t6 + RIGHT JOIN + JSON_TABLE('[1, 2, 3]', 'lax $[2]' PASSING a AS parameter_name COLUMNS(o FOR ORDINALITY)) + ON TRUE + """) + .hasErrorCode(INVALID_COLUMN_REFERENCE) + .hasMessage("line 4:48: LATERAL reference not allowed in RIGHT JOIN"); + } + + @Test + public void testSubqueryInJsonTable() + { + analyze(""" + SELECT * + FROM JSON_TABLE( + (SELECT '[1, 2, 3]'), + 'lax $[2]' PASSING (SELECT 1) AS parameter_name + COLUMNS( + x BIGINT DEFAULT (SELECT 2) ON EMPTY)) + """); + } + + @Test + public void testAggregationInJsonTable() + { + assertFails(""" + SELECT * + FROM JSON_TABLE( + CAST(sum(1) AS varchar), + 'lax $' PASSING 2 AS parameter_name + COLUMNS( + x BIGINT DEFAULT 3 ON EMPTY DEFAULT 4 ON ERROR)) + """) + .hasErrorCode(EXPRESSION_NOT_SCALAR) + .hasMessage("line 3:5: JSON_TABLE input expression cannot contain aggregations, window functions or grouping operations: [sum(1)]"); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '1', + 'lax $' PASSING avg(2) AS parameter_name + COLUMNS( + x BIGINT DEFAULT 3 ON EMPTY DEFAULT 4 ON ERROR)) + """) + .hasErrorCode(EXPRESSION_NOT_SCALAR) + .hasMessage("line 4:21: JSON_TABLE path parameter cannot contain aggregations, window functions or grouping operations: [avg(2)]"); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '1', + 'lax $' PASSING 2 AS parameter_name + COLUMNS( + x BIGINT DEFAULT min(3) ON EMPTY DEFAULT 4 ON ERROR)) + """) + .hasErrorCode(EXPRESSION_NOT_SCALAR) + .hasMessage("line 6:26: default expression for JSON_TABLE column cannot contain aggregations, window functions or grouping operations: [min(3)]"); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '1', + 'lax $' PASSING 2 AS parameter_name + COLUMNS( + x BIGINT DEFAULT 3 ON EMPTY DEFAULT max(4) ON ERROR)) + """) + .hasErrorCode(EXPRESSION_NOT_SCALAR) + .hasMessage("line 6:45: default expression for JSON_TABLE column cannot contain aggregations, window functions or grouping operations: [max(4)]"); + } + + @Test + public void testAliasJsonTable() + { + analyze(""" + SELECT t.y + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS(x BIGINT)) t(y) + """); + + analyze(""" + SELECT t.x + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS(x BIGINT)) t + """); } @Test diff --git a/core/trino-parser/src/main/java/io/trino/sql/jsonpath/PathParser.java b/core/trino-parser/src/main/java/io/trino/sql/jsonpath/PathParser.java index 2e54556165d6..cfbea2a262f7 100644 --- a/core/trino-parser/src/main/java/io/trino/sql/jsonpath/PathParser.java +++ b/core/trino-parser/src/main/java/io/trino/sql/jsonpath/PathParser.java @@ -40,13 +40,13 @@ public final class PathParser { private final BaseErrorListener errorListener; - public PathParser(Location startLocation) + public static PathParser withRelativeErrorLocation(Location startLocation) { requireNonNull(startLocation, "startLocation is null"); int pathStartLine = startLocation.line; int pathStartColumn = startLocation.column; - this.errorListener = new BaseErrorListener() + return new PathParser(new BaseErrorListener() { @Override public void syntaxError(Recognizer recognizer, Object offendingSymbol, int line, int charPositionInLine, String message, RecognitionException e) @@ -58,7 +58,26 @@ public void syntaxError(Recognizer recognizer, Object offendingSymbol, int int columnInQuery = line == 1 ? pathStartColumn + 1 + charPositionInLine : charPositionInLine + 1; throw new ParsingException(message, e, lineInQuery, columnInQuery); } - }; + }); + } + + public static PathParser withConstantErrorLocation(Location location) + { + requireNonNull(location, "location is null"); + + return new PathParser(new BaseErrorListener() + { + @Override + public void syntaxError(Recognizer recognizer, Object offendingSymbol, int line, int charPositionInLine, String message, RecognitionException e) + { + throw new ParsingException(message, e, location.line, location.column); + } + }); + } + + private PathParser(BaseErrorListener errorListener) + { + this.errorListener = requireNonNull(errorListener, "errorListener is null"); } public PathNode parseJsonPath(String path) diff --git a/core/trino-parser/src/test/java/io/trino/sql/jsonpath/TestPathParser.java b/core/trino-parser/src/test/java/io/trino/sql/jsonpath/TestPathParser.java index f294b2299d69..b645c75c0b3e 100644 --- a/core/trino-parser/src/test/java/io/trino/sql/jsonpath/TestPathParser.java +++ b/core/trino-parser/src/test/java/io/trino/sql/jsonpath/TestPathParser.java @@ -75,7 +75,7 @@ public class TestPathParser { - private static final PathParser PATH_PARSER = new PathParser(new Location(1, 0)); + private static final PathParser PATH_PARSER = PathParser.withRelativeErrorLocation(new Location(1, 0)); private static final RecursiveComparisonConfiguration COMPARISON_CONFIGURATION = RecursiveComparisonConfiguration.builder().withStrictTypeChecking(true).build(); @Test diff --git a/core/trino-spi/src/main/java/io/trino/spi/StandardErrorCode.java b/core/trino-spi/src/main/java/io/trino/spi/StandardErrorCode.java index e16f80af4ca9..2d9943c97699 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/StandardErrorCode.java +++ b/core/trino-spi/src/main/java/io/trino/spi/StandardErrorCode.java @@ -147,6 +147,9 @@ public enum StandardErrorCode INVALID_CHECK_CONSTRAINT(123, USER_ERROR), INVALID_CATALOG_PROPERTY(124, USER_ERROR), CATALOG_UNAVAILABLE(125, USER_ERROR), + DUPLICATE_COLUMN_OR_PATH_NAME(126, USER_ERROR), + MISSING_PATH_NAME(127, USER_ERROR), + INVALID_PLAN(128, USER_ERROR), GENERIC_INTERNAL_ERROR(65536, INTERNAL_ERROR), TOO_MANY_REQUESTS_FAILED(65537, INTERNAL_ERROR),