diff --git a/core/trino-main/src/main/java/io/trino/metadata/FunctionManager.java b/core/trino-main/src/main/java/io/trino/metadata/FunctionManager.java index 23e071d2ecc2..9a7c208380ea 100644 --- a/core/trino-main/src/main/java/io/trino/metadata/FunctionManager.java +++ b/core/trino-main/src/main/java/io/trino/metadata/FunctionManager.java @@ -324,7 +324,10 @@ private record FunctionKey(ResolvedFunction resolvedFunction, InvocationConventi public static FunctionManager createTestingFunctionManager() { TypeOperators typeOperators = new TypeOperators(); - GlobalFunctionCatalog functionCatalog = new GlobalFunctionCatalog(); + GlobalFunctionCatalog functionCatalog = new GlobalFunctionCatalog( + () -> { throw new UnsupportedOperationException(); }, + () -> { throw new UnsupportedOperationException(); }, + () -> { throw new UnsupportedOperationException(); }); functionCatalog.addFunctions(SystemFunctionBundle.create(new FeaturesConfig(), typeOperators, new BlockTypeOperators(typeOperators), UNKNOWN)); functionCatalog.addFunctions(new InternalFunctionBundle(new LiteralFunction(new InternalBlockEncodingSerde(new BlockEncodingManager(), TESTING_TYPE_MANAGER)))); return new FunctionManager(CatalogServiceProvider.fail(), functionCatalog, LanguageFunctionProvider.DISABLED); diff --git a/core/trino-main/src/main/java/io/trino/metadata/GlobalFunctionCatalog.java b/core/trino-main/src/main/java/io/trino/metadata/GlobalFunctionCatalog.java index 81056644011f..71a8e17eb5b8 100644 --- a/core/trino-main/src/main/java/io/trino/metadata/GlobalFunctionCatalog.java +++ b/core/trino-main/src/main/java/io/trino/metadata/GlobalFunctionCatalog.java @@ -18,9 +18,12 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Multimap; import com.google.errorprone.annotations.ThreadSafe; +import com.google.inject.Inject; +import com.google.inject.Provider; import io.trino.connector.system.GlobalSystemConnector; import io.trino.operator.table.ExcludeColumns.ExcludeColumnsFunctionHandle; import io.trino.operator.table.Sequence.SequenceFunctionHandle; +import io.trino.operator.table.json.JsonTable.JsonTableFunctionHandle; import io.trino.spi.function.AggregationFunctionMetadata; import io.trino.spi.function.AggregationImplementation; import io.trino.spi.function.BoundSignature; @@ -37,6 +40,7 @@ import io.trino.spi.function.WindowFunctionSupplier; import io.trino.spi.function.table.ConnectorTableFunctionHandle; import io.trino.spi.function.table.TableFunctionProcessorProvider; +import io.trino.spi.type.TypeManager; import io.trino.spi.type.TypeSignature; import java.util.Collection; @@ -53,19 +57,33 @@ import static io.trino.metadata.OperatorNameUtil.unmangleOperator; import static io.trino.operator.table.ExcludeColumns.getExcludeColumnsFunctionProcessorProvider; import static io.trino.operator.table.Sequence.getSequenceFunctionProcessorProvider; +import static io.trino.operator.table.json.JsonTable.getJsonTableFunctionProcessorProvider; import static io.trino.spi.function.FunctionKind.AGGREGATE; import static io.trino.spi.type.BigintType.BIGINT; import static io.trino.spi.type.BooleanType.BOOLEAN; import static io.trino.spi.type.IntegerType.INTEGER; import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; @ThreadSafe public class GlobalFunctionCatalog implements FunctionProvider { public static final String BUILTIN_SCHEMA = "builtin"; + + private final Provider metadata; + private final Provider typeManager; + private final Provider functionManager; private volatile FunctionMap functions = new FunctionMap(); + @Inject + public GlobalFunctionCatalog(Provider metadata, Provider typeManager, Provider functionManager) + { + this.metadata = requireNonNull(metadata, "metadata is null"); + this.typeManager = requireNonNull(typeManager, "typeManager is null"); + this.functionManager = requireNonNull(functionManager, "functionManager is null"); + } + public final synchronized void addFunctions(FunctionBundle functionBundle) { for (FunctionMetadata functionMetadata : functionBundle.getFunctions()) { @@ -187,6 +205,9 @@ public TableFunctionProcessorProvider getTableFunctionProcessorProvider(Connecto if (functionHandle instanceof SequenceFunctionHandle) { return getSequenceFunctionProcessorProvider(); } + if (functionHandle instanceof JsonTableFunctionHandle) { + return getJsonTableFunctionProcessorProvider(metadata.get(), typeManager.get(), functionManager.get()); + } return null; } diff --git a/core/trino-main/src/main/java/io/trino/metadata/MetadataManager.java b/core/trino-main/src/main/java/io/trino/metadata/MetadataManager.java index a96e0977e6d8..9c9221e38ebf 100644 --- a/core/trino-main/src/main/java/io/trino/metadata/MetadataManager.java +++ b/core/trino-main/src/main/java/io/trino/metadata/MetadataManager.java @@ -2761,7 +2761,10 @@ public MetadataManager build() GlobalFunctionCatalog globalFunctionCatalog = this.globalFunctionCatalog; if (globalFunctionCatalog == null) { - globalFunctionCatalog = new GlobalFunctionCatalog(); + globalFunctionCatalog = new GlobalFunctionCatalog( + () -> { throw new UnsupportedOperationException(); }, + () -> { throw new UnsupportedOperationException(); }, + () -> { throw new UnsupportedOperationException(); }); TypeOperators typeOperators = new TypeOperators(); globalFunctionCatalog.addFunctions(SystemFunctionBundle.create(new FeaturesConfig(), typeOperators, new BlockTypeOperators(typeOperators), UNKNOWN)); globalFunctionCatalog.addFunctions(new InternalFunctionBundle(new LiteralFunction(new InternalBlockEncodingSerde(new BlockEncodingManager(), typeManager)))); diff --git a/core/trino-main/src/main/java/io/trino/operator/TableFunctionOperator.java b/core/trino-main/src/main/java/io/trino/operator/TableFunctionOperator.java index f309a6d145c5..7b41bde101ed 100644 --- a/core/trino-main/src/main/java/io/trino/operator/TableFunctionOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/TableFunctionOperator.java @@ -128,7 +128,6 @@ public TableFunctionOperatorFactory( { requireNonNull(planNodeId, "planNodeId is null"); requireNonNull(tableFunctionProvider, "tableFunctionProvider is null"); - requireNonNull(catalogHandle, "catalogHandle is null"); requireNonNull(functionHandle, "functionHandle is null"); requireNonNull(requiredChannels, "requiredChannels is null"); requireNonNull(markerChannels, "markerChannels is null"); @@ -272,6 +271,7 @@ public TableFunctionOperator( this.operatorContext = operatorContext; this.session = operatorContext.getSession().toConnectorSession(catalogHandle); + this.processEmptyInput = !pruneWhenEmpty; PagesIndex pagesIndex = pagesIndexFactory.newPagesIndex(sourceTypes, expectedPositions); diff --git a/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTable.java b/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTable.java new file mode 100644 index 000000000000..3b64b58b34f8 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTable.java @@ -0,0 +1,215 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.table.json; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableList; +import io.trino.metadata.FunctionManager; +import io.trino.metadata.Metadata; +import io.trino.operator.table.json.execution.JsonTableProcessingFragment; +import io.trino.spi.Page; +import io.trino.spi.PageBuilder; +import io.trino.spi.block.SqlRow; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.function.table.ConnectorTableFunctionHandle; +import io.trino.spi.function.table.TableFunctionDataProcessor; +import io.trino.spi.function.table.TableFunctionProcessorProvider; +import io.trino.spi.function.table.TableFunctionProcessorState; +import io.trino.spi.type.RowType; +import io.trino.spi.type.Type; +import io.trino.spi.type.TypeManager; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import static com.google.common.collect.Iterables.getOnlyElement; +import static io.trino.operator.scalar.json.ParameterUtil.getParametersArray; +import static io.trino.operator.table.json.execution.ExecutionPlanner.getExecutionPlan; +import static io.trino.spi.function.table.TableFunctionProcessorState.Finished.FINISHED; +import static io.trino.spi.function.table.TableFunctionProcessorState.Processed.produced; +import static io.trino.spi.function.table.TableFunctionProcessorState.Processed.usedInput; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.TypeUtils.readNativeValue; +import static io.trino.spi.type.TypeUtils.writeNativeValue; +import static io.trino.type.Json2016Type.JSON_2016; +import static java.util.Objects.requireNonNull; + +/** + * Implements feature ISO/IEC 9075-2:2023(E) 7.11 'JSON table' + * including features T824, T827, T838 + */ +public class JsonTable +{ + private JsonTable() {} + + /** + * This class comprises all information necessary to execute the json_table function: + * + * @param processingPlan the root of the processing plan tree + * @param outer the parent-child relationship between the input relation and the processingPlan result + * @param errorOnError the error behavior: true for ERROR ON ERROR, false for EMPTY ON ERROR + * @param parametersType type of the row containing JSON path parameters for the root JSON path. The function expects the parameters row in the channel 1. + * Other channels in the input page correspond to JSON context item (channel 0), and default values for the value columns. Each value column in the processingPlan + * knows the indexes of its default channels. + * @param outputTypes types of the proper columns produced by the function + */ + public record JsonTableFunctionHandle(JsonTablePlanNode processingPlan, boolean outer, boolean errorOnError, RowType parametersType, Type[] outputTypes) + implements ConnectorTableFunctionHandle + { + public JsonTableFunctionHandle + { + requireNonNull(processingPlan, "processingPlan is null"); + requireNonNull(parametersType, "parametersType is null"); + requireNonNull(outputTypes, "outputTypes is null"); + } + } + + public static TableFunctionProcessorProvider getJsonTableFunctionProcessorProvider(Metadata metadata, TypeManager typeManager, FunctionManager functionManager) + { + return new TableFunctionProcessorProvider() + { + @Override + public TableFunctionDataProcessor getDataProcessor(ConnectorSession session, ConnectorTableFunctionHandle handle) + { + JsonTableFunctionHandle jsonTableFunctionHandle = (JsonTableFunctionHandle) handle; + Object[] newRow = new Object[jsonTableFunctionHandle.outputTypes().length]; + JsonTableProcessingFragment executionPlan = getExecutionPlan( + jsonTableFunctionHandle.processingPlan(), + newRow, + jsonTableFunctionHandle.errorOnError(), + jsonTableFunctionHandle.outputTypes(), + session, + metadata, + typeManager, + functionManager); + return new JsonTableFunctionProcessor(executionPlan, newRow, jsonTableFunctionHandle.outputTypes(), jsonTableFunctionHandle.parametersType(), jsonTableFunctionHandle.outer()); + } + }; + } + + public static class JsonTableFunctionProcessor + implements TableFunctionDataProcessor + { + private final PageBuilder pageBuilder; + private final int properColumnsCount; + private final JsonTableProcessingFragment executionPlan; + private final Object[] newRow; + private final RowType parametersType; + private final boolean outer; + + private long totalPositionsProcessed; + private int currentPosition = -1; + private boolean currentPositionAlreadyProduced; + + public JsonTableFunctionProcessor(JsonTableProcessingFragment executionPlan, Object[] newRow, Type[] outputTypes, RowType parametersType, boolean outer) + { + this.pageBuilder = new PageBuilder(ImmutableList.builder() + .add(outputTypes) + .add(BIGINT) // add additional position for pass-through index + .build()); + this.properColumnsCount = outputTypes.length; + this.executionPlan = requireNonNull(executionPlan, "executionPlan is null"); + this.newRow = requireNonNull(newRow, "newRow is null"); + this.parametersType = requireNonNull(parametersType, "parametersType is null"); + this.outer = outer; + } + + @Override + public TableFunctionProcessorState process(List> input) + { + // no more input pages + if (input == null) { + if (pageBuilder.isEmpty()) { + return FINISHED; + } + return flushPageBuilder(); + } + + Page inputPage = getOnlyElement(input).orElseThrow(); + while (!pageBuilder.isFull()) { + // new input page + if (currentPosition == -1) { + if (inputPage.getPositionCount() == 0) { + return usedInput(); + } + else { + currentPosition = 0; + currentPositionAlreadyProduced = false; + totalPositionsProcessed++; + SqlRow parametersRow = (SqlRow) readNativeValue(parametersType, inputPage.getBlock(1), currentPosition); + executionPlan.resetRoot( + (JsonNode) readNativeValue(JSON_2016, inputPage.getBlock(0), currentPosition), + inputPage, + currentPosition, + getParametersArray(parametersType, parametersRow)); + } + } + + // try to get output row for the current position (one position can produce multiple rows) + boolean gotNewRow = executionPlan.getRow(); + if (gotNewRow) { + currentPositionAlreadyProduced = true; + addOutputRow(); + } + else { + if (outer && !currentPositionAlreadyProduced) { + addNullPaddedRow(); + } + // go to next position in the input page + currentPosition++; + if (currentPosition < inputPage.getPositionCount()) { + currentPositionAlreadyProduced = false; + totalPositionsProcessed++; + SqlRow parametersRow = (SqlRow) readNativeValue(parametersType, inputPage.getBlock(1), currentPosition); + executionPlan.resetRoot( + (JsonNode) readNativeValue(JSON_2016, inputPage.getBlock(0), currentPosition), + inputPage, + currentPosition, + getParametersArray(parametersType, parametersRow)); + } + else { + currentPosition = -1; + return usedInput(); + } + } + } + + return flushPageBuilder(); + } + + private TableFunctionProcessorState flushPageBuilder() + { + TableFunctionProcessorState result = produced(pageBuilder.build()); + pageBuilder.reset(); + return result; + } + + private void addOutputRow() + { + pageBuilder.declarePosition(); + for (int channel = 0; channel < properColumnsCount; channel++) { + writeNativeValue(pageBuilder.getType(channel), pageBuilder.getBlockBuilder(channel), newRow[channel]); + } + // pass-through index from partition start + BIGINT.writeLong(pageBuilder.getBlockBuilder(properColumnsCount), totalPositionsProcessed - 1); + } + + private void addNullPaddedRow() + { + Arrays.fill(newRow, null); + addOutputRow(); + } + } +} diff --git a/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTableColumn.java b/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTableColumn.java new file mode 100644 index 000000000000..8727e4254c67 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTableColumn.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.table.json; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "@type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = JsonTableOrdinalityColumn.class, name = "ordinality"), + @JsonSubTypes.Type(value = JsonTableQueryColumn.class, name = "query"), + @JsonSubTypes.Type(value = JsonTableValueColumn.class, name = "value"), +}) + +public sealed interface JsonTableColumn + permits JsonTableOrdinalityColumn, JsonTableQueryColumn, JsonTableValueColumn +{ +} diff --git a/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTableOrdinalityColumn.java b/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTableOrdinalityColumn.java new file mode 100644 index 000000000000..904bb385e442 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTableOrdinalityColumn.java @@ -0,0 +1,19 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.table.json; + +public record JsonTableOrdinalityColumn(int outputIndex) + implements JsonTableColumn +{ +} diff --git a/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTablePlanCross.java b/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTablePlanCross.java new file mode 100644 index 000000000000..f61c13f920c9 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTablePlanCross.java @@ -0,0 +1,30 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.table.json; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +import static com.google.common.base.Preconditions.checkArgument; + +public record JsonTablePlanCross(List siblings) + implements JsonTablePlanNode +{ + public JsonTablePlanCross(List siblings) + { + this.siblings = ImmutableList.copyOf(siblings); + checkArgument(siblings.size() >= 2, "less than 2 siblings in Cross node"); + } +} diff --git a/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTablePlanLeaf.java b/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTablePlanLeaf.java new file mode 100644 index 000000000000..f1cbafbe86ce --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTablePlanLeaf.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.table.json; + +import com.google.common.collect.ImmutableList; +import io.trino.json.ir.IrJsonPath; + +import java.util.List; + +import static java.util.Objects.requireNonNull; + +public record JsonTablePlanLeaf(IrJsonPath path, List columns) + implements JsonTablePlanNode +{ + public JsonTablePlanLeaf(IrJsonPath path, List columns) + { + this.path = requireNonNull(path, "path is null"); + this.columns = ImmutableList.copyOf(columns); + } +} diff --git a/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTablePlanNode.java b/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTablePlanNode.java new file mode 100644 index 000000000000..73b56a75fb17 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTablePlanNode.java @@ -0,0 +1,32 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.table.json; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "@type") +@JsonSubTypes({ + @JsonSubTypes.Type(value = JsonTablePlanCross.class, name = "cross"), + @JsonSubTypes.Type(value = JsonTablePlanLeaf.class, name = "leaf"), + @JsonSubTypes.Type(value = JsonTablePlanSingle.class, name = "single"), + @JsonSubTypes.Type(value = JsonTablePlanUnion.class, name = "union"), +}) + +public sealed interface JsonTablePlanNode + permits JsonTablePlanCross, JsonTablePlanLeaf, JsonTablePlanSingle, JsonTablePlanUnion +{ +} diff --git a/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTablePlanSingle.java b/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTablePlanSingle.java new file mode 100644 index 000000000000..49423e2c4bd2 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTablePlanSingle.java @@ -0,0 +1,33 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.table.json; + +import com.google.common.collect.ImmutableList; +import io.trino.json.ir.IrJsonPath; + +import java.util.List; + +import static java.util.Objects.requireNonNull; + +public record JsonTablePlanSingle(IrJsonPath path, List columns, boolean outer, JsonTablePlanNode child) + implements JsonTablePlanNode +{ + public JsonTablePlanSingle(IrJsonPath path, List columns, boolean outer, JsonTablePlanNode child) + { + this.path = requireNonNull(path, "path is null"); + this.columns = ImmutableList.copyOf(columns); + this.outer = outer; + this.child = requireNonNull(child, "child is null"); + } +} diff --git a/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTablePlanUnion.java b/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTablePlanUnion.java new file mode 100644 index 000000000000..e8a1f1caeaf4 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTablePlanUnion.java @@ -0,0 +1,30 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.table.json; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +import static com.google.common.base.Preconditions.checkArgument; + +public record JsonTablePlanUnion(List siblings) + implements JsonTablePlanNode +{ + public JsonTablePlanUnion(List siblings) + { + this.siblings = ImmutableList.copyOf(siblings); + checkArgument(siblings.size() >= 2, "less than 2 siblings in Union node"); + } +} diff --git a/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTableQueryColumn.java b/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTableQueryColumn.java new file mode 100644 index 000000000000..117df03c2c25 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTableQueryColumn.java @@ -0,0 +1,40 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.table.json; + +import io.trino.json.ir.IrJsonPath; +import io.trino.metadata.ResolvedFunction; + +import static java.util.Objects.requireNonNull; + +/** + * This representation does not contain all properties of the column as specified in json_table invocation. + * Certain properties are handled by the output function which is applied later. + * These are: output format and quotes behavior. + */ +public record JsonTableQueryColumn( + int outputIndex, + ResolvedFunction function, + IrJsonPath path, + long wrapperBehavior, + long emptyBehavior, + long errorBehavior) + implements JsonTableColumn +{ + public JsonTableQueryColumn + { + requireNonNull(function, "function is null"); + requireNonNull(path, "path is null"); + } +} diff --git a/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTableValueColumn.java b/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTableValueColumn.java new file mode 100644 index 000000000000..6d87bc4a5ffd --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/table/json/JsonTableValueColumn.java @@ -0,0 +1,36 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.table.json; + +import io.trino.json.ir.IrJsonPath; +import io.trino.metadata.ResolvedFunction; + +import static java.util.Objects.requireNonNull; + +public record JsonTableValueColumn( + int outputIndex, + ResolvedFunction function, + IrJsonPath path, + long emptyBehavior, + int emptyDefaultInput, // channel number or -1 when default not specified + long errorBehavior, + int errorDefaultInput) // channel number or -1 when default not specified + implements JsonTableColumn +{ + public JsonTableValueColumn + { + requireNonNull(function, "function is null"); + requireNonNull(path, "path is null"); + } +} diff --git a/core/trino-main/src/main/java/io/trino/operator/table/json/execution/Column.java b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/Column.java new file mode 100644 index 000000000000..15eab03d10d3 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/Column.java @@ -0,0 +1,24 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.table.json.execution; + +import com.fasterxml.jackson.databind.JsonNode; +import io.trino.spi.Page; + +public interface Column +{ + Object evaluate(long sequentialNumber, JsonNode item, Page input, int position); + + int getOutputIndex(); +} diff --git a/core/trino-main/src/main/java/io/trino/operator/table/json/execution/ExecutionPlanner.java b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/ExecutionPlanner.java new file mode 100644 index 000000000000..e6c4879db057 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/ExecutionPlanner.java @@ -0,0 +1,159 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.table.json.execution; + +import com.google.common.collect.ImmutableList; +import io.trino.json.JsonPathInvocationContext; +import io.trino.metadata.FunctionManager; +import io.trino.metadata.Metadata; +import io.trino.operator.table.json.JsonTableColumn; +import io.trino.operator.table.json.JsonTableOrdinalityColumn; +import io.trino.operator.table.json.JsonTablePlanCross; +import io.trino.operator.table.json.JsonTablePlanLeaf; +import io.trino.operator.table.json.JsonTablePlanNode; +import io.trino.operator.table.json.JsonTablePlanSingle; +import io.trino.operator.table.json.JsonTablePlanUnion; +import io.trino.operator.table.json.JsonTableQueryColumn; +import io.trino.operator.table.json.JsonTableValueColumn; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.function.InvocationConvention; +import io.trino.spi.function.ScalarFunctionImplementation; +import io.trino.spi.type.Type; +import io.trino.spi.type.TypeManager; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Throwables.throwIfUnchecked; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.trino.spi.function.InvocationConvention.InvocationArgumentConvention.BOXED_NULLABLE; +import static io.trino.spi.function.InvocationConvention.InvocationArgumentConvention.NEVER_NULL; +import static io.trino.spi.function.InvocationConvention.InvocationReturnConvention.NULLABLE_RETURN; + +public class ExecutionPlanner +{ + private ExecutionPlanner() + { + } + + public static JsonTableProcessingFragment getExecutionPlan( + JsonTablePlanNode plan, + Object[] newRow, + boolean errorOnError, + Type[] outputTypes, + ConnectorSession session, + Metadata metadata, + TypeManager typeManager, + FunctionManager functionManager) + { + if (plan instanceof JsonTablePlanLeaf planLeaf) { + return new FragmentLeaf( + planLeaf.path(), + planLeaf.columns().stream() + .map(column -> getColumn(column, outputTypes, session, functionManager)) + .collect(toImmutableList()), + errorOnError, + newRow, + session, + metadata, + typeManager, + functionManager); + } + if (plan instanceof JsonTablePlanSingle planSingle) { + return new FragmentSingle( + planSingle.path(), + planSingle.columns().stream() + .map(column -> getColumn(column, outputTypes, session, functionManager)) + .collect(toImmutableList()), + errorOnError, + planSingle.outer(), + getExecutionPlan(planSingle.child(), newRow, errorOnError, outputTypes, session, metadata, typeManager, functionManager), + newRow, + session, + metadata, + typeManager, + functionManager); + } + if (plan instanceof JsonTablePlanCross planCross) { + return new FragmentCross(planCross.siblings().stream() + .map(sibling -> getExecutionPlan(sibling, newRow, errorOnError, outputTypes, session, metadata, typeManager, functionManager)) + .collect(toImmutableList())); + } + JsonTablePlanUnion planUnion = (JsonTablePlanUnion) plan; + return new FragmentUnion( + planUnion.siblings().stream() + .map(sibling -> getExecutionPlan(sibling, newRow, errorOnError, outputTypes, session, metadata, typeManager, functionManager)) + .collect(toImmutableList()), + newRow); + } + + private static Column getColumn(JsonTableColumn column, Type[] outputTypes, ConnectorSession session, FunctionManager functionManager) + { + if (column instanceof JsonTableValueColumn valueColumn) { + ScalarFunctionImplementation implementation = functionManager.getScalarFunctionImplementation( + valueColumn.function(), + new InvocationConvention( + ImmutableList.of(BOXED_NULLABLE, BOXED_NULLABLE, BOXED_NULLABLE, NEVER_NULL, BOXED_NULLABLE, NEVER_NULL, BOXED_NULLABLE), + NULLABLE_RETURN, + true, + true)); + JsonPathInvocationContext context; + checkArgument(implementation.getInstanceFactory().isPresent(), "instance factory is missing"); + try { + context = (JsonPathInvocationContext) implementation.getInstanceFactory().get().invoke(); + } + catch (Throwable throwable) { + throwIfUnchecked(throwable); + throw new RuntimeException(throwable); + } + return new ValueColumn( + valueColumn.outputIndex(), + implementation.getMethodHandle() + .bindTo(context) + .bindTo(session), + valueColumn.path(), + valueColumn.emptyBehavior(), + valueColumn.emptyDefaultInput(), + valueColumn.errorBehavior(), + valueColumn.errorDefaultInput(), + outputTypes[valueColumn.outputIndex()]); + } + if (column instanceof JsonTableQueryColumn queryColumn) { + ScalarFunctionImplementation implementation = functionManager.getScalarFunctionImplementation( + queryColumn.function(), + new InvocationConvention( + ImmutableList.of(BOXED_NULLABLE, BOXED_NULLABLE, BOXED_NULLABLE, NEVER_NULL, NEVER_NULL, NEVER_NULL), + NULLABLE_RETURN, + true, + true)); + JsonPathInvocationContext context; + checkArgument(implementation.getInstanceFactory().isPresent(), "instance factory is missing"); + try { + context = (JsonPathInvocationContext) implementation.getInstanceFactory().get().invoke(); + } + catch (Throwable throwable) { + throwIfUnchecked(throwable); + throw new RuntimeException(throwable); + } + return new QueryColumn( + queryColumn.outputIndex(), + implementation.getMethodHandle() + .bindTo(context) + .bindTo(session), + queryColumn.path(), + queryColumn.wrapperBehavior(), + queryColumn.emptyBehavior(), + queryColumn.errorBehavior()); + } + return new OrdinalityColumn(((JsonTableOrdinalityColumn) column).outputIndex()); + } +} diff --git a/core/trino-main/src/main/java/io/trino/operator/table/json/execution/FragmentCross.java b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/FragmentCross.java new file mode 100644 index 000000000000..56cbdbe724be --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/FragmentCross.java @@ -0,0 +1,93 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.table.json.execution; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableList; +import io.trino.spi.Page; + +import java.util.Arrays; +import java.util.List; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class FragmentCross + implements JsonTableProcessingFragment +{ + private final List siblings; + private final int[] outputLayout; + + private Page input; + private int position; + private JsonNode currentItem; + private int currentSiblingIndex; + + public FragmentCross(List siblings) + { + this.siblings = ImmutableList.copyOf(siblings); + checkArgument(siblings.size() >= 2, "less than 2 siblings in Cross node"); + this.outputLayout = siblings.stream() + .map(JsonTableProcessingFragment::getOutputLayout) + .flatMapToInt(Arrays::stream) + .toArray(); + } + + @Override + public void reset(JsonNode item, Page input, int position) + { + this.currentItem = requireNonNull(item, "item is null"); + this.input = requireNonNull(input, "input is null"); + this.position = position; + siblings.get(0).reset(item, input, position); + this.currentSiblingIndex = 0; + } + + /** + * All values produced by the siblings are stored on corresponding positions in `newRow`. It is a temporary representation of the result row, and is shared by all Fragments. + * The values in `newRow` are not cleared between subsequent calls to getRow(), so that the parts which do not change are automatically reused. + */ + @Override + public boolean getRow() + { + while (currentSiblingIndex >= 0) { + boolean currentSiblingProducedRow = siblings.get(currentSiblingIndex).getRow(); + if (currentSiblingProducedRow) { + for (int i = currentSiblingIndex + 1; i < siblings.size(); i++) { + JsonTableProcessingFragment sibling = siblings.get(i); + sibling.reset(currentItem, input, position); + boolean siblingProducedRow = sibling.getRow(); + if (!siblingProducedRow) { + // if any sibling is empty, the whole CROSS fragment is empty + return false; + } + } + currentSiblingIndex = siblings.size() - 1; + return true; + } + + // current sibling is finished + currentSiblingIndex--; + } + + // fragment is finished + return false; + } + + @Override + public int[] getOutputLayout() + { + return outputLayout; + } +} diff --git a/core/trino-main/src/main/java/io/trino/operator/table/json/execution/FragmentLeaf.java b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/FragmentLeaf.java new file mode 100644 index 000000000000..9a11e63067d4 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/FragmentLeaf.java @@ -0,0 +1,109 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.table.json.execution; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableList; +import io.trino.json.JsonPathEvaluator; +import io.trino.json.ir.IrJsonPath; +import io.trino.metadata.FunctionManager; +import io.trino.metadata.Metadata; +import io.trino.spi.Page; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.type.TypeManager; + +import java.util.List; + +import static io.trino.operator.table.json.execution.SequenceEvaluator.getSequence; +import static java.util.Objects.requireNonNull; + +public class FragmentLeaf + implements JsonTableProcessingFragment +{ + private static final Object[] NO_PARAMETERS = new Object[0]; + + private final JsonPathEvaluator pathEvaluator; + private final List columns; + private final boolean errorOnError; + private final int[] outputLayout; + + // the place where the computed values (or nulls) are stored while computing an output row + private final Object[] newRow; + + private Page input; + private int position; + private List sequence; + private int nextItemIndex; + + public FragmentLeaf( + IrJsonPath path, + List columns, + boolean errorOnError, + Object[] newRow, + ConnectorSession session, + Metadata metadata, + TypeManager typeManager, + FunctionManager functionManager) + { + requireNonNull(path, "path is null"); + this.pathEvaluator = new JsonPathEvaluator(path, session, metadata, typeManager, functionManager); + this.columns = ImmutableList.copyOf(columns); + this.errorOnError = errorOnError; + this.outputLayout = columns.stream() + .mapToInt(Column::getOutputIndex) + .toArray(); + this.newRow = requireNonNull(newRow, "newRow is null"); + } + + @Override + public void reset(JsonNode item, Page input, int position) + { + resetRoot(item, input, position, NO_PARAMETERS); + } + + /** + * FragmentLeaf can be the root Fragment. The root fragment is the only fragment that may have path parameters. + * Prepares the root Fragment to produce rows for the new JSON item and a set of path parameters. + */ + @Override + public void resetRoot(JsonNode item, Page input, int position, Object[] pathParameters) + { + requireNonNull(pathParameters, "pathParameters is null"); + this.input = requireNonNull(input, "input is null"); + this.position = position; + this.nextItemIndex = 0; + this.sequence = getSequence(item, pathParameters, pathEvaluator, errorOnError); + } + + @Override + public boolean getRow() + { + if (nextItemIndex >= sequence.size()) { + // fragment is finished + return false; + } + JsonNode currentItem = sequence.get(nextItemIndex); + nextItemIndex++; // it is correct to pass the updated value to `column.evaluate()` because ordinality numbers are 1-based according to ISO/IEC 9075-2:2016(E) 7.11 p.461 General rules. + for (Column column : columns) { + newRow[column.getOutputIndex()] = column.evaluate(nextItemIndex, currentItem, input, position); + } + return true; + } + + @Override + public int[] getOutputLayout() + { + return outputLayout; + } +} diff --git a/core/trino-main/src/main/java/io/trino/operator/table/json/execution/FragmentSingle.java b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/FragmentSingle.java new file mode 100644 index 000000000000..d3d285f0658e --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/FragmentSingle.java @@ -0,0 +1,156 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.table.json.execution; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableList; +import io.trino.json.JsonPathEvaluator; +import io.trino.json.ir.IrJsonPath; +import io.trino.metadata.FunctionManager; +import io.trino.metadata.Metadata; +import io.trino.spi.Page; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.type.TypeManager; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.IntStream; + +import static io.trino.operator.table.json.execution.SequenceEvaluator.getSequence; +import static java.util.Objects.requireNonNull; + +public class FragmentSingle + implements JsonTableProcessingFragment +{ + private static final Object[] NO_PARAMETERS = new Object[] {}; + + private final JsonPathEvaluator pathEvaluator; + private final List columns; + private final boolean errorOnError; + private final boolean outer; + private final JsonTableProcessingFragment child; + private final int[] outputLayout; + + // the place where the computed values (or nulls) are stored while computing an output row + private final Object[] newRow; + + private Page input; + private int position; + private List sequence; + private int nextItemIndex; + + // start processing next item from the sequence + private boolean processNextItem; + + // indicates if we need to produce null-padded row for OUTER + private boolean childAlreadyProduced; + + public FragmentSingle( + IrJsonPath path, + List columns, + boolean errorOnError, + boolean outer, + JsonTableProcessingFragment child, + Object[] newRow, + ConnectorSession session, + Metadata metadata, + TypeManager typeManager, + FunctionManager functionManager) + { + requireNonNull(path, "path is null"); + this.pathEvaluator = new JsonPathEvaluator(path, session, metadata, typeManager, functionManager); + this.columns = ImmutableList.copyOf(columns); + this.errorOnError = errorOnError; + this.outer = outer; + this.child = requireNonNull(child, "child is null"); + this.outputLayout = IntStream.concat( + columns.stream() + .mapToInt(Column::getOutputIndex), + Arrays.stream(child.getOutputLayout())) + .toArray(); + this.newRow = requireNonNull(newRow, "newRow is null"); + } + + @Override + public void reset(JsonNode item, Page input, int position) + { + resetRoot(item, input, position, NO_PARAMETERS); + } + + /** + * FragmentSingle can be the root Fragment. The root fragment is the only fragment that may have path parameters. + * Prepares the root Fragment to produce rows for the new JSON item and a set of path parameters. + */ + @Override + public void resetRoot(JsonNode item, Page input, int position, Object[] pathParameters) + { + requireNonNull(pathParameters, "pathParameters is null"); + this.input = requireNonNull(input, "input is null"); + this.position = position; + this.nextItemIndex = 0; + this.processNextItem = true; + this.sequence = getSequence(item, pathParameters, pathEvaluator, errorOnError); + } + + /** + * All values produced by the columns are stored on corresponding positions in `newRow`. + * The values in `newRow` are not cleared between subsequent calls to `getRow()`, so the values for columns are automatically reused during iterating over child. + */ + @Override + public boolean getRow() + { + while (true) { + if (processNextItem) { + if (nextItemIndex >= sequence.size()) { + // fragment is finished + return false; + } + JsonNode currentItem = sequence.get(nextItemIndex); + nextItemIndex++; // it is correct to pass the updated value to `column.evaluate()` because ordinality numbers are 1-based according to ISO/IEC 9075-2:2016(E) 7.11 p.461 General rules. + for (Column column : columns) { + newRow[column.getOutputIndex()] = column.evaluate(nextItemIndex, currentItem, input, position); + } + child.reset(currentItem, input, position); + childAlreadyProduced = false; + processNextItem = false; + } + + boolean childProducedRow = child.getRow(); + if (childProducedRow) { + childAlreadyProduced = true; + return true; + } + + // child is finished + processNextItem = true; + if (outer && !childAlreadyProduced) { + appendNulls(child); + return true; + } + } + } + + private void appendNulls(JsonTableProcessingFragment fragment) + { + for (int column : fragment.getOutputLayout()) { + newRow[column] = null; + } + } + + @Override + public int[] getOutputLayout() + { + return outputLayout; + } +} diff --git a/core/trino-main/src/main/java/io/trino/operator/table/json/execution/FragmentUnion.java b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/FragmentUnion.java new file mode 100644 index 000000000000..30ae142f9dfa --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/FragmentUnion.java @@ -0,0 +1,96 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.table.json.execution; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableList; +import io.trino.spi.Page; + +import java.util.Arrays; +import java.util.List; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class FragmentUnion + implements JsonTableProcessingFragment +{ + private final List siblings; + private final int[] outputLayout; + + // the place where the computed values (or nulls) are stored while computing an output row + private final Object[] newRow; + + private int currentSiblingIndex; + + public FragmentUnion(List siblings, Object[] newRow) + { + this.siblings = ImmutableList.copyOf(siblings); + checkArgument(siblings.size() >= 2, "less than 2 siblings in Union node"); + this.outputLayout = siblings.stream() + .map(JsonTableProcessingFragment::getOutputLayout) + .flatMapToInt(Arrays::stream) + .toArray(); + this.newRow = requireNonNull(newRow, "newRow is null"); + } + + @Override + public void reset(JsonNode item, Page input, int position) + { + requireNonNull(item, "item is null"); + requireNonNull(input, "input is null"); + siblings.stream() + .forEach(sibling -> sibling.reset(item, input, position)); + this.currentSiblingIndex = 0; + appendNulls(this); + } + + /** + * The values produced by the current sibling are stored on corresponding positions in `newRow`, and for other siblings `newRow` is filled with nulls. + * The values in `newRow` are not cleared between subsequent calls to getRow(), so that the parts which do not change are automatically reused. + */ + @Override + public boolean getRow() + { + while (true) { + if (currentSiblingIndex >= siblings.size()) { + // fragment is finished + return false; + } + + JsonTableProcessingFragment currentSibling = siblings.get(currentSiblingIndex); + boolean currentSiblingProducedRow = currentSibling.getRow(); + if (currentSiblingProducedRow) { + return true; + } + + // current sibling is finished + appendNulls(currentSibling); + currentSiblingIndex++; + } + } + + private void appendNulls(JsonTableProcessingFragment fragment) + { + for (int column : fragment.getOutputLayout()) { + newRow[column] = null; + } + } + + @Override + public int[] getOutputLayout() + { + return outputLayout; + } +} diff --git a/core/trino-main/src/main/java/io/trino/operator/table/json/execution/JsonTableProcessingFragment.java b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/JsonTableProcessingFragment.java new file mode 100644 index 000000000000..bfe518b41036 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/JsonTableProcessingFragment.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.table.json.execution; + +import com.fasterxml.jackson.databind.JsonNode; +import io.trino.spi.Page; + +public interface JsonTableProcessingFragment +{ + /** + * Prepares the Fragment to produce rows for the new JSON item. + * Note: This method must be called for each new JSON item. Due to nesting, there might be multiple JSON items to process for a single position in the input page. + * Therefore, input and position may not change for subsequent calls. + * + * @param item the new JSON item + * @param input the input Page currently processed by json_table function + * @param position the currently processed position in the input page + */ + void reset(JsonNode item, Page input, int position); + + /** + * Prepares the root Fragment to produce rows for the new JSON item and new set of path parameters. + * Note: at the root level, there is one JSON item and one set of path parameters to process for each position in the input page. + * + * @param item the new JSON item + * @param input the input Page currently processed by json_table function + * @param position the currently processed position in the input page + * @param pathParameters JSON path parameters for the top-level JSON path + */ + default void resetRoot(JsonNode item, Page input, int position, Object[] pathParameters) + { + throw new IllegalStateException("not the root fragment"); + } + + /** + * Tries to produce output values for all columns included in the Fragment, + * and stores them in corresponding positions in `newRow`. + * Note: According to OUTER or UNION semantics, some values might be null-padded instead of computed. + * Note: a single JSON item might result in multiple output rows. To fully process a JSON item, the caller must: + * - reset the Fragment with the JSON item + * - call getRow() and collect output rows as long as `true` is returned + * If `false` is returned, there is no output row available, and the JSON item is fully processed + * + * @return true if row was produced, false if row was not produced (Fragment is finished) + */ + boolean getRow(); + + /** + * Returns an array containing indexes of columns produced by the fragment within all columns produced by json_table. + */ + int[] getOutputLayout(); +} diff --git a/core/trino-main/src/main/java/io/trino/operator/table/json/execution/OrdinalityColumn.java b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/OrdinalityColumn.java new file mode 100644 index 000000000000..d26479ecf9e4 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/OrdinalityColumn.java @@ -0,0 +1,40 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.table.json.execution; + +import com.fasterxml.jackson.databind.JsonNode; +import io.trino.spi.Page; + +public class OrdinalityColumn + implements Column +{ + private final int outputIndex; + + public OrdinalityColumn(int outputIndex) + { + this.outputIndex = outputIndex; + } + + @Override + public Object evaluate(long sequentialNumber, JsonNode item, Page input, int position) + { + return sequentialNumber; + } + + @Override + public int getOutputIndex() + { + return outputIndex; + } +} diff --git a/core/trino-main/src/main/java/io/trino/operator/table/json/execution/QueryColumn.java b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/QueryColumn.java new file mode 100644 index 000000000000..613ec5c41db3 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/QueryColumn.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.table.json.execution; + +import com.fasterxml.jackson.databind.JsonNode; +import io.trino.json.ir.IrJsonPath; +import io.trino.spi.Page; + +import java.lang.invoke.MethodHandle; + +import static com.google.common.base.Throwables.throwIfUnchecked; +import static java.util.Objects.requireNonNull; + +public class QueryColumn + implements Column +{ + private final int outputIndex; + private final MethodHandle methodHandle; + private final IrJsonPath path; + private final long wrapperBehavior; + private final long emptyBehavior; + private final long errorBehavior; + + public QueryColumn(int outputIndex, MethodHandle methodHandle, IrJsonPath path, long wrapperBehavior, long emptyBehavior, long errorBehavior) + { + this.outputIndex = outputIndex; + this.methodHandle = requireNonNull(methodHandle, "methodHandle is null"); + this.path = requireNonNull(path, "path is null"); + this.wrapperBehavior = wrapperBehavior; + this.emptyBehavior = emptyBehavior; + this.errorBehavior = errorBehavior; + } + + @Override + public Object evaluate(long sequentialNumber, JsonNode item, Page input, int position) + { + try { + return methodHandle.invoke(item, path, null, wrapperBehavior, emptyBehavior, errorBehavior); + } + catch (Throwable throwable) { + // According to ISO/IEC 9075-2:2016(E) 7.11 p.462 General rules 1) e) ii) 3) D) any exception thrown by column evaluation should be propagated. + throwIfUnchecked(throwable); + throw new RuntimeException(throwable); + } + } + + @Override + public int getOutputIndex() + { + return outputIndex; + } +} diff --git a/core/trino-main/src/main/java/io/trino/operator/table/json/execution/SequenceEvaluator.java b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/SequenceEvaluator.java new file mode 100644 index 000000000000..32b4fe0b9389 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/SequenceEvaluator.java @@ -0,0 +1,93 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.table.json.execution; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableList; +import io.trino.json.JsonPathEvaluator; +import io.trino.json.PathEvaluationException; +import io.trino.json.ir.TypedValue; +import io.trino.operator.scalar.json.JsonOutputConversionException; + +import java.util.List; +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkState; +import static io.trino.json.JsonInputErrorNode.JSON_ERROR; +import static io.trino.json.ir.SqlJsonLiteralConverter.getJsonNode; +import static java.lang.String.format; + +public class SequenceEvaluator +{ + private SequenceEvaluator() + { + } + + // creates a sequence of JSON items, and applies error handling + public static List getSequence(JsonNode item, Object[] pathParameters, JsonPathEvaluator pathEvaluator, boolean errorOnError) + { + if (item == null) { + // According to ISO/IEC 9075-2:2016(E) 7.11 p.461 General rules 1) a) empty table should be returned for null input. Empty sequence will result in an empty table. + return ImmutableList.of(); + } + // According to ISO/IEC 9075-2:2016(E) 7.11 p.461 General rules 1) e) exception thrown by path evaluation should be handled accordingly to json_table error behavior (ERROR or EMPTY). + // handle input conversion error for the context item + if (item.equals(JSON_ERROR)) { + checkState(!errorOnError, "input conversion error should have been thrown in the input function"); + // the error behavior is EMPTY ON ERROR. Empty sequence will result in an empty table. + return ImmutableList.of(); + } + // handle input conversion error for the path parameters + for (Object parameter : pathParameters) { + if (parameter.equals(JSON_ERROR)) { + checkState(!errorOnError, "input conversion error should have been thrown in the input function"); + // the error behavior is EMPTY ON ERROR. Empty sequence will result in an empty table. + return ImmutableList.of(); + } + } + // evaluate path into a sequence + List pathResult; + try { + pathResult = pathEvaluator.evaluate(item, pathParameters); + } + catch (PathEvaluationException e) { + if (errorOnError) { + throw e; + } + // the error behavior is EMPTY ON ERROR. Empty sequence will result in an empty table. + return ImmutableList.of(); + } + // convert sequence to JSON items + ImmutableList.Builder builder = ImmutableList.builder(); + for (Object element : pathResult) { + if (element instanceof TypedValue typedValue) { + Optional jsonNode = getJsonNode(typedValue); + if (jsonNode.isEmpty()) { + if (errorOnError) { + throw new JsonOutputConversionException(format( + "JSON path returned a scalar SQL value of type %s that cannot be represented as JSON", + ((TypedValue) element).getType())); + } + // the error behavior is EMPTY ON ERROR. Empty sequence will result in an empty table. + return ImmutableList.of(); + } + builder.add(jsonNode.get()); + } + else { + builder.add((JsonNode) element); + } + } + return builder.build(); + } +} diff --git a/core/trino-main/src/main/java/io/trino/operator/table/json/execution/ValueColumn.java b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/ValueColumn.java new file mode 100644 index 000000000000..a8c29d8baff7 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/operator/table/json/execution/ValueColumn.java @@ -0,0 +1,93 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.operator.table.json.execution; + +import com.fasterxml.jackson.databind.JsonNode; +import io.trino.json.ir.IrJsonPath; +import io.trino.spi.Page; +import io.trino.spi.type.Type; + +import java.lang.invoke.MethodHandle; + +import static com.google.common.base.Throwables.throwIfUnchecked; +import static io.trino.spi.type.TypeUtils.readNativeValue; +import static java.util.Objects.requireNonNull; + +public class ValueColumn + implements Column +{ + private final int outputIndex; + private final MethodHandle methodHandle; + private final IrJsonPath path; + private final long emptyBehavior; + private final int emptyDefaultInput; + private final long errorBehavior; + private final int errorDefaultInput; + private final Type resultType; + + public ValueColumn( + int outputIndex, + MethodHandle methodHandle, + IrJsonPath path, + long emptyBehavior, + int emptyDefaultInput, + long errorBehavior, + int errorDefaultInput, + Type resultType) + { + this.outputIndex = outputIndex; + this.methodHandle = requireNonNull(methodHandle, "methodHandle is null"); + this.path = requireNonNull(path, "path is null"); + this.emptyBehavior = emptyBehavior; + this.emptyDefaultInput = emptyDefaultInput; + this.errorBehavior = errorBehavior; + this.errorDefaultInput = errorDefaultInput; + this.resultType = requireNonNull(resultType, "resultType is null"); + } + + @Override + public Object evaluate(long sequentialNumber, JsonNode item, Page input, int position) + { + Object emptyDefault; + if (emptyDefaultInput == -1) { + emptyDefault = null; + } + else { + emptyDefault = readNativeValue(resultType, input.getBlock(emptyDefaultInput), position); + } + + Object errorDefault; + if (errorDefaultInput == -1) { + errorDefault = null; + } + else { + errorDefault = readNativeValue(resultType, input.getBlock(errorDefaultInput), position); + } + + try { + return methodHandle.invoke(item, path, null, emptyBehavior, emptyDefault, errorBehavior, errorDefault); + } + catch (Throwable throwable) { + // According to ISO/IEC 9075-2:2016(E) 7.11 p.462 General rules 1) e) ii) 2) D) any exception thrown by column evaluation should be propagated. + throwIfUnchecked(throwable); + throw new RuntimeException(throwable); + } + } + + @Override + public int getOutputIndex() + { + return outputIndex; + } +} diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/Analysis.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/Analysis.java index 96b29a1b31cd..c878a5b11858 100644 --- a/core/trino-main/src/main/java/io/trino/sql/analyzer/Analysis.java +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/Analysis.java @@ -62,6 +62,8 @@ import io.trino.sql.tree.Identifier; import io.trino.sql.tree.InPredicate; import io.trino.sql.tree.Join; +import io.trino.sql.tree.JsonTable; +import io.trino.sql.tree.JsonTableColumnDefinition; import io.trino.sql.tree.LambdaArgumentDeclaration; import io.trino.sql.tree.MeasureDefinition; import io.trino.sql.tree.Node; @@ -161,9 +163,10 @@ public class Analysis private final Set> patternAggregations = new LinkedHashSet<>(); // for JSON features - private final Map, JsonPathAnalysis> jsonPathAnalyses = new LinkedHashMap<>(); + private final Map, JsonPathAnalysis> jsonPathAnalyses = new LinkedHashMap<>(); private final Map, ResolvedFunction> jsonInputFunctions = new LinkedHashMap<>(); - private final Map, ResolvedFunction> jsonOutputFunctions = new LinkedHashMap<>(); + private final Map, ResolvedFunction> jsonOutputFunctions = new LinkedHashMap<>(); + private final Map, JsonTableAnalysis> jsonTableAnalyses = new LinkedHashMap<>(); private final Map, List> aggregates = new LinkedHashMap<>(); private final Map, List> orderByAggregates = new LinkedHashMap<>(); @@ -204,7 +207,7 @@ public class Analysis private final Map, Type> sortKeyCoercionsForFrameBoundComparison = new LinkedHashMap<>(); private final Map, ResolvedFunction> frameBoundCalculations = new LinkedHashMap<>(); private final Map, List> relationCoercions = new LinkedHashMap<>(); - private final Map, RoutineEntry> resolvedFunctions = new LinkedHashMap<>(); + private final Map, RoutineEntry> resolvedFunctions = new LinkedHashMap<>(); private final Map, LambdaArgumentDeclaration> lambdaArgumentReferences = new LinkedHashMap<>(); private final Map columns = new LinkedHashMap<>(); @@ -656,12 +659,12 @@ public Set getResolvedFunctions() .collect(toImmutableSet()); } - public ResolvedFunction getResolvedFunction(Expression node) + public ResolvedFunction getResolvedFunction(Node node) { return resolvedFunctions.get(NodeRef.of(node)).getFunction(); } - public void addResolvedFunction(Expression node, ResolvedFunction function, String authorization) + public void addResolvedFunction(Node node, ResolvedFunction function, String authorization) { resolvedFunctions.put(NodeRef.of(node), new RoutineEntry(function, authorization)); } @@ -1021,14 +1024,19 @@ public boolean isPatternAggregation(FunctionCall function) return patternAggregations.contains(NodeRef.of(function)); } - public void setJsonPathAnalyses(Map, JsonPathAnalysis> pathAnalyses) + public void setJsonPathAnalyses(Map, JsonPathAnalysis> pathAnalyses) { jsonPathAnalyses.putAll(pathAnalyses); } - public JsonPathAnalysis getJsonPathAnalysis(Expression expression) + public void setJsonPathAnalysis(Node node, JsonPathAnalysis pathAnalysis) { - return jsonPathAnalyses.get(NodeRef.of(expression)); + jsonPathAnalyses.put(NodeRef.of(node), pathAnalysis); + } + + public JsonPathAnalysis getJsonPathAnalysis(Node node) + { + return jsonPathAnalyses.get(NodeRef.of(node)); } public void setJsonInputFunctions(Map, ResolvedFunction> functions) @@ -1041,14 +1049,24 @@ public ResolvedFunction getJsonInputFunction(Expression expression) return jsonInputFunctions.get(NodeRef.of(expression)); } - public void setJsonOutputFunctions(Map, ResolvedFunction> functions) + public void setJsonOutputFunctions(Map, ResolvedFunction> functions) { jsonOutputFunctions.putAll(functions); } - public ResolvedFunction getJsonOutputFunction(Expression expression) + public ResolvedFunction getJsonOutputFunction(Node node) + { + return jsonOutputFunctions.get(NodeRef.of(node)); + } + + public void addJsonTableAnalysis(JsonTable jsonTable, JsonTableAnalysis analysis) + { + jsonTableAnalyses.put(NodeRef.of(jsonTable), analysis); + } + + public JsonTableAnalysis getJsonTableAnalysis(JsonTable jsonTable) { - return jsonOutputFunctions.get(NodeRef.of(expression)); + return jsonTableAnalyses.get(NodeRef.of(jsonTable)); } public Map>> getTableColumnReferences() @@ -2388,4 +2406,19 @@ public ConnectorTransactionHandle getTransactionHandle() return transactionHandle; } } + + public record JsonTableAnalysis( + CatalogHandle catalogHandle, + ConnectorTransactionHandle transactionHandle, + RowType parametersType, + List> orderedOutputColumns) + { + public JsonTableAnalysis + { + requireNonNull(catalogHandle, "catalogHandle is null"); + requireNonNull(transactionHandle, "transactionHandle is null"); + requireNonNull(parametersType, "parametersType is null"); + requireNonNull(orderedOutputColumns, "orderedOutputColumns is null"); + } + } } diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/ExpressionAnalyzer.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/ExpressionAnalyzer.java index 0ef7db9ef384..c97fcd62bd4d 100644 --- a/core/trino-main/src/main/java/io/trino/sql/analyzer/ExpressionAnalyzer.java +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/ExpressionAnalyzer.java @@ -78,6 +78,7 @@ import io.trino.sql.tree.CurrentSchema; import io.trino.sql.tree.CurrentTime; import io.trino.sql.tree.CurrentUser; +import io.trino.sql.tree.DataType; import io.trino.sql.tree.DecimalLiteral; import io.trino.sql.tree.DereferenceExpression; import io.trino.sql.tree.DoubleLiteral; @@ -106,6 +107,7 @@ import io.trino.sql.tree.JsonPathParameter; import io.trino.sql.tree.JsonPathParameter.JsonFormat; import io.trino.sql.tree.JsonQuery; +import io.trino.sql.tree.JsonTable; import io.trino.sql.tree.JsonValue; import io.trino.sql.tree.LambdaArgumentDeclaration; import io.trino.sql.tree.LambdaExpression; @@ -123,6 +125,7 @@ import io.trino.sql.tree.ProcessingMode; import io.trino.sql.tree.QualifiedName; import io.trino.sql.tree.QuantifiedComparisonExpression; +import io.trino.sql.tree.QueryColumn; import io.trino.sql.tree.RangeQuantifier; import io.trino.sql.tree.Row; import io.trino.sql.tree.RowPattern; @@ -139,6 +142,7 @@ import io.trino.sql.tree.TimestampLiteral; import io.trino.sql.tree.Trim; import io.trino.sql.tree.TryExpression; +import io.trino.sql.tree.ValueColumn; import io.trino.sql.tree.VariableDefinition; import io.trino.sql.tree.WhenClause; import io.trino.sql.tree.WindowFrame; @@ -297,7 +301,7 @@ public class ExpressionAnalyzer // Cache from SQL type name to Type; every Type in the cache has a CAST defined from VARCHAR private final Cache varcharCastableTypeCache = buildNonEvictableCache(CacheBuilder.newBuilder().maximumSize(1000)); - private final Map, ResolvedFunction> resolvedFunctions = new LinkedHashMap<>(); + private final Map, ResolvedFunction> resolvedFunctions = new LinkedHashMap<>(); private final Set> subqueries = new LinkedHashSet<>(); private final Set> existsSubqueries = new LinkedHashSet<>(); private final Map, Type> expressionCoercions = new LinkedHashMap<>(); @@ -336,9 +340,9 @@ public class ExpressionAnalyzer private final Set> patternAggregations = new LinkedHashSet<>(); // for JSON functions - private final Map, JsonPathAnalysis> jsonPathAnalyses = new LinkedHashMap<>(); + private final Map, JsonPathAnalysis> jsonPathAnalyses = new LinkedHashMap<>(); private final Map, ResolvedFunction> jsonInputFunctions = new LinkedHashMap<>(); - private final Map, ResolvedFunction> jsonOutputFunctions = new LinkedHashMap<>(); + private final Map, ResolvedFunction> jsonOutputFunctions = new LinkedHashMap<>(); private final Session session; private final Map, Expression> parameters; @@ -402,7 +406,7 @@ private ExpressionAnalyzer( this.functionResolver = plannerContext.getFunctionResolver(warningCollector); } - public Map, ResolvedFunction> getResolvedFunctions() + public Map, ResolvedFunction> getResolvedFunctions() { return unmodifiableMap(resolvedFunctions); } @@ -500,6 +504,42 @@ private Type analyze(Expression expression, Scope baseScope, Context context) return visitor.process(expression, new StackableAstVisitor.StackableAstVisitorContext<>(context)); } + private RowType analyzeJsonPathInvocation(JsonTable node, Scope scope, CorrelationSupport correlationSupport) + { + Visitor visitor = new Visitor(scope, warningCollector); + List inputTypes = visitor.analyzeJsonPathInvocation("JSON_TABLE", node, node.getJsonPathInvocation(), new StackableAstVisitor.StackableAstVisitorContext<>(Context.notInLambda(scope, correlationSupport))); + return (RowType) inputTypes.get(2); + } + + private Type analyzeJsonValueExpression(ValueColumn column, JsonPathAnalysis pathAnalysis, Scope scope, CorrelationSupport correlationSupport) + { + Visitor visitor = new Visitor(scope, warningCollector); + List pathInvocationArgumentTypes = ImmutableList.of(JSON_2016, plannerContext.getTypeManager().getType(TypeId.of(JsonPath2016Type.NAME)), JSON_NO_PARAMETERS_ROW_TYPE); + return visitor.analyzeJsonValueExpression( + column, + pathAnalysis, + Optional.of(column.getType()), + pathInvocationArgumentTypes, + column.getEmptyBehavior(), + column.getEmptyDefault(), + column.getErrorBehavior(), + column.getErrorDefault(), + new StackableAstVisitor.StackableAstVisitorContext<>(Context.notInLambda(scope, correlationSupport))); + } + + private Type analyzeJsonQueryExpression(QueryColumn column, Scope scope) + { + Visitor visitor = new Visitor(scope, warningCollector); + List pathInvocationArgumentTypes = ImmutableList.of(JSON_2016, plannerContext.getTypeManager().getType(TypeId.of(JsonPath2016Type.NAME)), JSON_NO_PARAMETERS_ROW_TYPE); + return visitor.analyzeJsonQueryExpression( + column, + column.getWrapperBehavior(), + column.getQuotesBehavior(), + pathInvocationArgumentTypes, + Optional.of(column.getType()), + Optional.of(column.getFormat())); + } + private void analyzeWindow(ResolvedWindow window, Scope scope, Node originalNode, CorrelationSupport correlationSupport) { Visitor visitor = new Visitor(scope, warningCollector); @@ -566,7 +606,7 @@ public Set> getPatternAggregations() return patternAggregations; } - public Map, JsonPathAnalysis> getJsonPathAnalyses() + public Map, JsonPathAnalysis> getJsonPathAnalyses() { return jsonPathAnalyses; } @@ -576,7 +616,7 @@ public Map, ResolvedFunction> getJsonInputFunctions() return jsonInputFunctions; } - public Map, ResolvedFunction> getJsonOutputFunctions() + public Map, ResolvedFunction> getJsonOutputFunctions() { return jsonOutputFunctions; } @@ -2532,15 +2572,38 @@ public Type visitJsonExists(JsonExists node, StackableAstVisitorContext public Type visitJsonValue(JsonValue node, StackableAstVisitorContext context) { List pathInvocationArgumentTypes = analyzeJsonPathInvocation("JSON_VALUE", node, node.getJsonPathInvocation(), context); + Type returnedType = analyzeJsonValueExpression( + node, + jsonPathAnalyses.get(NodeRef.of(node)), + node.getReturnedType(), + pathInvocationArgumentTypes, + node.getEmptyBehavior(), + node.getEmptyDefault(), + Optional.of(node.getErrorBehavior()), + node.getErrorDefault(), + context); + return setExpressionType(node, returnedType); + } + private Type analyzeJsonValueExpression( + Node node, + JsonPathAnalysis pathAnalysis, + Optional declaredReturnedType, + List pathInvocationArgumentTypes, + JsonValue.EmptyOrErrorBehavior emptyBehavior, + Optional declaredEmptyDefault, + Optional errorBehavior, + Optional declaredErrorDefault, + StackableAstVisitorContext context) + { // validate returned type Type returnedType = VARCHAR; // default - if (node.getReturnedType().isPresent()) { + if (declaredReturnedType.isPresent()) { try { - returnedType = plannerContext.getTypeManager().getType(toTypeSignature(node.getReturnedType().get())); + returnedType = plannerContext.getTypeManager().getType(toTypeSignature(declaredReturnedType.get())); } catch (TypeNotFoundException e) { - throw semanticException(TYPE_MISMATCH, node, "Unknown type: %s", node.getReturnedType().get()); + throw semanticException(TYPE_MISMATCH, node, "Unknown type: %s", declaredReturnedType.get()); } } @@ -2550,10 +2613,9 @@ public Type visitJsonValue(JsonValue node, StackableAstVisitorContext c !isDateTimeType(returnedType) || returnedType.equals(INTERVAL_DAY_TIME) || returnedType.equals(INTERVAL_YEAR_MONTH)) { - throw semanticException(TYPE_MISMATCH, node, "Invalid return type of function JSON_VALUE: %s", node.getReturnedType().get()); + throw semanticException(TYPE_MISMATCH, node, "Invalid return type of function JSON_VALUE: %s", declaredReturnedType.get()); } - JsonPathAnalysis pathAnalysis = jsonPathAnalyses.get(NodeRef.of(node)); Type resultType = pathAnalysis.getType(pathAnalysis.getPath()); if (resultType != null && !resultType.equals(returnedType)) { try { @@ -2565,20 +2627,23 @@ public Type visitJsonValue(JsonValue node, StackableAstVisitorContext c } // validate default values for empty and error behavior - if (node.getEmptyDefault().isPresent()) { - Expression emptyDefault = node.getEmptyDefault().get(); - if (node.getEmptyBehavior() != DEFAULT) { - throw semanticException(INVALID_FUNCTION_ARGUMENT, emptyDefault, "Default value specified for %s ON EMPTY behavior", node.getEmptyBehavior()); + if (declaredEmptyDefault.isPresent()) { + Expression emptyDefault = declaredEmptyDefault.get(); + if (emptyBehavior != DEFAULT) { + throw semanticException(INVALID_FUNCTION_ARGUMENT, emptyDefault, "Default value specified for %s ON EMPTY behavior", emptyBehavior); } Type type = process(emptyDefault, context); // this would normally be done after function resolution, but we know that the default expression is always coerced to the returnedType coerceType(emptyDefault, type, returnedType, "Function JSON_VALUE default ON EMPTY result"); } - if (node.getErrorDefault().isPresent()) { - Expression errorDefault = node.getErrorDefault().get(); - if (node.getErrorBehavior() != DEFAULT) { - throw semanticException(INVALID_FUNCTION_ARGUMENT, errorDefault, "Default value specified for %s ON ERROR behavior", node.getErrorBehavior()); + if (declaredErrorDefault.isPresent()) { + Expression errorDefault = declaredErrorDefault.get(); + if (errorBehavior.isEmpty()) { + throw new IllegalStateException("error default specified without error behavior specified"); + } + if (errorBehavior.orElseThrow() != DEFAULT) { + throw semanticException(INVALID_FUNCTION_ARGUMENT, errorDefault, "Default value specified for %s ON ERROR behavior", errorBehavior.orElseThrow()); } Type type = process(errorDefault, context); // this would normally be done after function resolution, but we know that the default expression is always coerced to the returnedType @@ -2606,21 +2671,32 @@ public Type visitJsonValue(JsonValue node, StackableAstVisitorContext c throw new TrinoException(e::getErrorCode, extractLocation(node), e.getMessage(), e); } resolvedFunctions.put(NodeRef.of(node), function); - Type type = function.getSignature().getReturnType(); - return setExpressionType(node, type); + return function.getSignature().getReturnType(); } @Override public Type visitJsonQuery(JsonQuery node, StackableAstVisitorContext context) { List pathInvocationArgumentTypes = analyzeJsonPathInvocation("JSON_QUERY", node, node.getJsonPathInvocation(), context); + Type returnedType = analyzeJsonQueryExpression( + node, + node.getWrapperBehavior(), + node.getQuotesBehavior(), + pathInvocationArgumentTypes, + node.getReturnedType(), + node.getOutputFormat()); + return setExpressionType(node, returnedType); + } - // validate wrapper and quotes behavior - if ((node.getWrapperBehavior() == CONDITIONAL || node.getWrapperBehavior() == UNCONDITIONAL) && node.getQuotesBehavior().isPresent()) { - throw semanticException(INVALID_FUNCTION_ARGUMENT, node, "%s QUOTES behavior specified with WITH %s ARRAY WRAPPER behavior", node.getQuotesBehavior().get(), node.getWrapperBehavior()); - } - + private Type analyzeJsonQueryExpression( + Node node, + JsonQuery.ArrayWrapperBehavior wrapperBehavior, + Optional quotesBehavior, + List pathInvocationArgumentTypes, + Optional declaredReturnedType, + Optional declaredOutputFormat) + { // wrapper behavior, empty behavior and error behavior will be passed as arguments to function // quotes behavior is handled by the corresponding output function List argumentTypes = ImmutableList.builder() @@ -2630,6 +2706,11 @@ public Type visitJsonQuery(JsonQuery node, StackableAstVisitorContext c .add(TINYINT) // error behavior: enum encoded as integer value .build(); + // validate wrapper and quotes behavior + if ((wrapperBehavior == CONDITIONAL || wrapperBehavior == UNCONDITIONAL) && quotesBehavior.isPresent()) { + throw semanticException(INVALID_FUNCTION_ARGUMENT, node, "%s QUOTES behavior specified with WITH %s ARRAY WRAPPER behavior", quotesBehavior.get(), wrapperBehavior); + } + // resolve function ResolvedFunction function; try { @@ -2645,15 +2726,15 @@ public Type visitJsonQuery(JsonQuery node, StackableAstVisitorContext c // analyze returned type and format Type returnedType = VARCHAR; // default - if (node.getReturnedType().isPresent()) { + if (declaredReturnedType.isPresent()) { try { - returnedType = plannerContext.getTypeManager().getType(toTypeSignature(node.getReturnedType().get())); + returnedType = plannerContext.getTypeManager().getType(toTypeSignature(declaredReturnedType.get())); } catch (TypeNotFoundException e) { - throw semanticException(TYPE_MISMATCH, node, "Unknown type: %s", node.getReturnedType().get()); + throw semanticException(TYPE_MISMATCH, node, "Unknown type: %s", declaredReturnedType.get()); } } - JsonFormat outputFormat = node.getOutputFormat().orElse(JsonFormat.JSON); // default + JsonFormat outputFormat = declaredOutputFormat.orElse(JsonFormat.JSON); // default // resolve function to format output ResolvedFunction outputFunction = getOutputFunction(returnedType, outputFormat, node); @@ -2670,13 +2751,15 @@ public Type visitJsonQuery(JsonQuery node, StackableAstVisitorContext c } } - return setExpressionType(node, returnedType); + return returnedType; } - private List analyzeJsonPathInvocation(String functionName, Expression node, JsonPathInvocation jsonPathInvocation, StackableAstVisitorContext context) + private List analyzeJsonPathInvocation(String functionName, Node node, JsonPathInvocation jsonPathInvocation, StackableAstVisitorContext context) { jsonPathInvocation.getPathName().ifPresent(pathName -> { - throw semanticException(INVALID_PATH, pathName, "JSON path name is not allowed in %s function", functionName); + if (!(node instanceof JsonTable)) { + throw semanticException(INVALID_PATH, pathName, "JSON path name is not allowed in %s function", functionName); + } }); // ANALYZE THE CONTEXT ITEM @@ -3444,6 +3527,79 @@ public static ExpressionAnalysis analyzeExpression( analyzer.getWindowFunctions()); } + public static ParametersTypeAndAnalysis analyzeJsonPathInvocation( + JsonTable node, + Session session, + PlannerContext plannerContext, + StatementAnalyzerFactory statementAnalyzerFactory, + AccessControl accessControl, + Scope scope, + Analysis analysis, + WarningCollector warningCollector, + CorrelationSupport correlationSupport) + { + ExpressionAnalyzer analyzer = new ExpressionAnalyzer(plannerContext, accessControl, statementAnalyzerFactory, analysis, session, TypeProvider.empty(), warningCollector); + RowType parametersRowType = analyzer.analyzeJsonPathInvocation(node, scope, correlationSupport); + updateAnalysis(analysis, analyzer, session, accessControl); + return new ParametersTypeAndAnalysis( + parametersRowType, + new ExpressionAnalysis( + analyzer.getExpressionTypes(), + analyzer.getExpressionCoercions(), + analyzer.getSubqueryInPredicates(), + analyzer.getSubqueries(), + analyzer.getExistsSubqueries(), + analyzer.getColumnReferences(), + analyzer.getTypeOnlyCoercions(), + analyzer.getQuantifiedComparisons(), + analyzer.getWindowFunctions())); + } + + public record ParametersTypeAndAnalysis(RowType parametersType, ExpressionAnalysis expressionAnalysis) {} + + public static TypeAndAnalysis analyzeJsonValueExpression( + ValueColumn column, + JsonPathAnalysis pathAnalysis, + Session session, + PlannerContext plannerContext, + StatementAnalyzerFactory statementAnalyzerFactory, + AccessControl accessControl, + Scope scope, + Analysis analysis, + WarningCollector warningCollector, + CorrelationSupport correlationSupport) + { + ExpressionAnalyzer analyzer = new ExpressionAnalyzer(plannerContext, accessControl, statementAnalyzerFactory, analysis, session, TypeProvider.empty(), warningCollector); + Type type = analyzer.analyzeJsonValueExpression(column, pathAnalysis, scope, correlationSupport); + updateAnalysis(analysis, analyzer, session, accessControl); + return new TypeAndAnalysis(type, new ExpressionAnalysis( + analyzer.getExpressionTypes(), + analyzer.getExpressionCoercions(), + analyzer.getSubqueryInPredicates(), + analyzer.getSubqueries(), + analyzer.getExistsSubqueries(), + analyzer.getColumnReferences(), + analyzer.getTypeOnlyCoercions(), + analyzer.getQuantifiedComparisons(), + analyzer.getWindowFunctions())); + } + + public static Type analyzeJsonQueryExpression( + QueryColumn column, + Session session, + PlannerContext plannerContext, + StatementAnalyzerFactory statementAnalyzerFactory, + AccessControl accessControl, + Scope scope, + Analysis analysis, + WarningCollector warningCollector) + { + ExpressionAnalyzer analyzer = new ExpressionAnalyzer(plannerContext, accessControl, statementAnalyzerFactory, analysis, session, TypeProvider.empty(), warningCollector); + Type type = analyzer.analyzeJsonQueryExpression(column, scope); + updateAnalysis(analysis, analyzer, session, accessControl); + return type; + } + public static void analyzeExpressionWithoutSubqueries( Session session, PlannerContext plannerContext, @@ -3715,4 +3871,6 @@ public Optional getLabel() return label; } } + + public record TypeAndAnalysis(Type type, ExpressionAnalysis analysis) {} } diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/JsonPathAnalyzer.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/JsonPathAnalyzer.java index 093d9f016cbe..ef949ccd2527 100644 --- a/core/trino-main/src/main/java/io/trino/sql/analyzer/JsonPathAnalyzer.java +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/JsonPathAnalyzer.java @@ -59,6 +59,7 @@ import io.trino.sql.jsonpath.tree.StartsWithPredicate; import io.trino.sql.jsonpath.tree.TypeMethod; import io.trino.sql.tree.Node; +import io.trino.sql.tree.NodeLocation; import io.trino.sql.tree.StringLiteral; import java.util.LinkedHashMap; @@ -108,11 +109,18 @@ public JsonPathAnalysis analyzeJsonPath(StringLiteral path, Map pa Location pathStart = extractLocation(path) .map(location -> new Location(location.getLineNumber(), location.getColumnNumber())) .orElseThrow(() -> new IllegalStateException("missing NodeLocation in path")); - PathNode root = new PathParser(pathStart).parseJsonPath(path.getValue()); + PathNode root = PathParser.withRelativeErrorLocation(pathStart).parseJsonPath(path.getValue()); new Visitor(parameterTypes, path).process(root); return new JsonPathAnalysis((JsonPath) root, types, jsonParameters); } + public JsonPathAnalysis analyzeImplicitJsonPath(String path, NodeLocation location) + { + PathNode root = PathParser.withFixedErrorLocation(new Location(location.getLineNumber(), location.getColumnNumber())).parseJsonPath(path); + new Visitor(ImmutableMap.of(), new StringLiteral(path)).process(root); + return new JsonPathAnalysis((JsonPath) root, types, jsonParameters); + } + /** * This visitor determines and validates output types of PathNodes, whenever they can be deduced and represented as SQL types. * In some cases, the type of a PathNode can be determined without context. E.g., the `double()` method always returns DOUBLE. diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java index 3641b35579ee..4a935a59777c 100644 --- a/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java @@ -23,11 +23,13 @@ import com.google.common.collect.Iterables; import com.google.common.collect.ListMultimap; import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; import com.google.common.collect.Streams; import com.google.common.math.IntMath; import io.airlift.slice.Slice; import io.trino.Session; import io.trino.SystemSessionProperties; +import io.trino.connector.system.GlobalSystemConnector; import io.trino.execution.Column; import io.trino.execution.warnings.WarningCollector; import io.trino.metadata.AnalyzePropertyManager; @@ -104,6 +106,7 @@ import io.trino.sql.InterpretedFunctionInvoker; import io.trino.sql.PlannerContext; import io.trino.sql.analyzer.Analysis.GroupingSetAnalysis; +import io.trino.sql.analyzer.Analysis.JsonTableAnalysis; import io.trino.sql.analyzer.Analysis.MergeAnalysis; import io.trino.sql.analyzer.Analysis.ResolvedWindow; import io.trino.sql.analyzer.Analysis.SelectExpression; @@ -111,6 +114,9 @@ import io.trino.sql.analyzer.Analysis.TableArgumentAnalysis; import io.trino.sql.analyzer.Analysis.TableFunctionInvocationAnalysis; import io.trino.sql.analyzer.Analysis.UnnestAnalysis; +import io.trino.sql.analyzer.ExpressionAnalyzer.ParametersTypeAndAnalysis; +import io.trino.sql.analyzer.ExpressionAnalyzer.TypeAndAnalysis; +import io.trino.sql.analyzer.JsonPathAnalyzer.JsonPathAnalysis; import io.trino.sql.analyzer.PatternRecognitionAnalyzer.PatternRecognitionAnalysis; import io.trino.sql.analyzer.Scope.AsteriskedIdentifierChainBasis; import io.trino.sql.parser.ParsingException; @@ -173,7 +179,11 @@ import io.trino.sql.tree.JoinCriteria; import io.trino.sql.tree.JoinOn; import io.trino.sql.tree.JoinUsing; +import io.trino.sql.tree.JsonPathInvocation; +import io.trino.sql.tree.JsonPathParameter; import io.trino.sql.tree.JsonTable; +import io.trino.sql.tree.JsonTableColumnDefinition; +import io.trino.sql.tree.JsonTableSpecificPlan; import io.trino.sql.tree.Lateral; import io.trino.sql.tree.Limit; import io.trino.sql.tree.LongLiteral; @@ -184,16 +194,23 @@ import io.trino.sql.tree.MergeInsert; import io.trino.sql.tree.MergeUpdate; import io.trino.sql.tree.NaturalJoin; +import io.trino.sql.tree.NestedColumns; import io.trino.sql.tree.Node; +import io.trino.sql.tree.NodeLocation; import io.trino.sql.tree.NodeRef; import io.trino.sql.tree.Offset; import io.trino.sql.tree.OrderBy; +import io.trino.sql.tree.OrdinalityColumn; import io.trino.sql.tree.Parameter; import io.trino.sql.tree.PatternRecognitionRelation; +import io.trino.sql.tree.PlanLeaf; +import io.trino.sql.tree.PlanParentChild; +import io.trino.sql.tree.PlanSiblings; import io.trino.sql.tree.Prepare; import io.trino.sql.tree.Property; import io.trino.sql.tree.QualifiedName; import io.trino.sql.tree.Query; +import io.trino.sql.tree.QueryColumn; import io.trino.sql.tree.QueryPeriod; import io.trino.sql.tree.QuerySpecification; import io.trino.sql.tree.RefreshMaterializedView; @@ -227,6 +244,7 @@ import io.trino.sql.tree.SortItem; import io.trino.sql.tree.StartTransaction; import io.trino.sql.tree.Statement; +import io.trino.sql.tree.StringLiteral; import io.trino.sql.tree.SubqueryExpression; import io.trino.sql.tree.SubscriptExpression; import io.trino.sql.tree.Table; @@ -242,6 +260,7 @@ import io.trino.sql.tree.Update; import io.trino.sql.tree.UpdateAssignment; import io.trino.sql.tree.Use; +import io.trino.sql.tree.ValueColumn; import io.trino.sql.tree.Values; import io.trino.sql.tree.VariableDefinition; import io.trino.sql.tree.Window; @@ -294,6 +313,7 @@ import static io.trino.spi.StandardErrorCode.COLUMN_NOT_FOUND; import static io.trino.spi.StandardErrorCode.COLUMN_TYPE_UNKNOWN; import static io.trino.spi.StandardErrorCode.DUPLICATE_COLUMN_NAME; +import static io.trino.spi.StandardErrorCode.DUPLICATE_COLUMN_OR_PATH_NAME; import static io.trino.spi.StandardErrorCode.DUPLICATE_NAMED_QUERY; import static io.trino.spi.StandardErrorCode.DUPLICATE_PROPERTY; import static io.trino.spi.StandardErrorCode.DUPLICATE_RANGE_VARIABLE; @@ -312,6 +332,7 @@ import static io.trino.spi.StandardErrorCode.INVALID_LIMIT_CLAUSE; import static io.trino.spi.StandardErrorCode.INVALID_ORDER_BY; import static io.trino.spi.StandardErrorCode.INVALID_PARTITION_BY; +import static io.trino.spi.StandardErrorCode.INVALID_PLAN; import static io.trino.spi.StandardErrorCode.INVALID_RECURSIVE_REFERENCE; import static io.trino.spi.StandardErrorCode.INVALID_ROW_FILTER; import static io.trino.spi.StandardErrorCode.INVALID_TABLE_FUNCTION_INVOCATION; @@ -324,6 +345,7 @@ import static io.trino.spi.StandardErrorCode.MISSING_COLUMN_NAME; import static io.trino.spi.StandardErrorCode.MISSING_GROUP_BY; import static io.trino.spi.StandardErrorCode.MISSING_ORDER_BY; +import static io.trino.spi.StandardErrorCode.MISSING_PATH_NAME; import static io.trino.spi.StandardErrorCode.MISSING_RETURN_TYPE; import static io.trino.spi.StandardErrorCode.NESTED_RECURSIVE; import static io.trino.spi.StandardErrorCode.NESTED_ROW_PATTERN_RECOGNITION; @@ -363,6 +385,8 @@ import static io.trino.sql.analyzer.AggregationAnalyzer.verifySourceAggregations; import static io.trino.sql.analyzer.Analyzer.verifyNoAggregateWindowOrGroupingFunctions; import static io.trino.sql.analyzer.CanonicalizationAware.canonicalizationAwareKey; +import static io.trino.sql.analyzer.ExpressionAnalyzer.analyzeJsonQueryExpression; +import static io.trino.sql.analyzer.ExpressionAnalyzer.analyzeJsonValueExpression; import static io.trino.sql.analyzer.ExpressionAnalyzer.createConstantAnalyzer; import static io.trino.sql.analyzer.ExpressionTreeUtils.asQualifiedName; import static io.trino.sql.analyzer.ExpressionTreeUtils.extractAggregateFunctions; @@ -3250,6 +3274,17 @@ protected Scope visitJoin(Join node, Optional scope) } } } + else if (isJsonTable(node.getRight())) { + if (criteria != null) { + if (!(criteria instanceof JoinOn) || !((JoinOn) criteria).getExpression().equals(TRUE_LITERAL)) { + throw semanticException( + NOT_SUPPORTED, + criteria instanceof JoinOn ? ((JoinOn) criteria).getExpression() : node, + "%s JOIN involving JSON_TABLE is only supported with condition ON TRUE", + node.getType().name()); + } + } + } else if (node.getType() == FULL) { if (!(criteria instanceof JoinOn) || !((JoinOn) criteria).getExpression().equals(TRUE_LITERAL)) { throw semanticException( @@ -3776,7 +3811,7 @@ private boolean isLateralRelation(Relation node) if (node instanceof AliasedRelation) { return isLateralRelation(((AliasedRelation) node).getRelation()); } - return node instanceof Unnest || node instanceof Lateral; + return node instanceof Unnest || node instanceof Lateral || node instanceof JsonTable; } private boolean isUnnestRelation(Relation node) @@ -3787,6 +3822,14 @@ private boolean isUnnestRelation(Relation node) return node instanceof Unnest; } + private boolean isJsonTable(Relation node) + { + if (node instanceof AliasedRelation) { + return isJsonTable(((AliasedRelation) node).getRelation()); + } + return node instanceof JsonTable; + } + @Override protected Scope visitValues(Values node, Optional scope) { @@ -3862,9 +3905,254 @@ else if (actualType instanceof RowType) { } @Override - protected Scope visitJsonTable(JsonTable node, Optional context) + protected Scope visitJsonTable(JsonTable node, Optional scope) + { + Scope enclosingScope = createScope(scope); + + // analyze the context item, the root JSON path, and the path parameters + RowType parametersType = analyzeJsonPathInvocation(node, enclosingScope); + + // json_table is implemented as a table function provided by the global catalog. + CatalogHandle catalogHandle = getRequiredCatalogHandle(metadata, session, node, GlobalSystemConnector.NAME); + ConnectorTransactionHandle transactionHandle = transactionManager.getConnectorTransaction(session.getRequiredTransactionId(), catalogHandle); + + // all column and path names must be unique + Set uniqueNames = new HashSet<>(); + JsonPathInvocation rootPath = node.getJsonPathInvocation(); + rootPath.getPathName().ifPresent(name -> uniqueNames.add(name.getCanonicalValue())); + + ImmutableList.Builder outputFields = ImmutableList.builder(); + ImmutableList.Builder> orderedOutputColumns = ImmutableList.builder(); + analyzeJsonTableColumns(node.getColumns(), uniqueNames, outputFields, orderedOutputColumns, enclosingScope, node); + + analysis.addJsonTableAnalysis(node, new JsonTableAnalysis(catalogHandle, transactionHandle, parametersType, orderedOutputColumns.build())); + + node.getPlan().ifPresent(plan -> { + if (plan instanceof JsonTableSpecificPlan specificPlan) { + validateJsonTableSpecificPlan(rootPath, specificPlan, node.getColumns()); + } + else { + // if PLAN DEFAULT is specified, all nested paths should be named + checkAllNestedPathsNamed(node.getColumns()); + } + }); + + return createAndAssignScope(node, scope, outputFields.build()); + } + + private RowType analyzeJsonPathInvocation(JsonTable node, Scope scope) + { + verifyNoAggregateWindowOrGroupingFunctions(session, functionResolver, accessControl, node.getJsonPathInvocation().getInputExpression(), "JSON_TABLE input expression"); + node.getJsonPathInvocation().getPathParameters().stream() + .map(JsonPathParameter::getParameter) + .forEach(parameter -> verifyNoAggregateWindowOrGroupingFunctions(session, functionResolver, accessControl, parameter, "JSON_TABLE path parameter")); + + ParametersTypeAndAnalysis parametersTypeAndAnalysis = ExpressionAnalyzer.analyzeJsonPathInvocation( + node, + session, + plannerContext, + statementAnalyzerFactory, + accessControl, + scope, + analysis, + WarningCollector.NOOP, + correlationSupport); + // context item and passed path parameters can contain subqueries - the subqueries are recorded under the enclosing JsonTable node + analysis.recordSubqueries(node, parametersTypeAndAnalysis.expressionAnalysis()); + return parametersTypeAndAnalysis.parametersType(); + } + + private void analyzeJsonTableColumns( + List columns, + Set uniqueNames, + ImmutableList.Builder outputFields, + ImmutableList.Builder> orderedOutputColumns, + Scope enclosingScope, + JsonTable jsonTable) + { + for (JsonTableColumnDefinition column : columns) { + if (column instanceof OrdinalityColumn ordinalityColumn) { + String name = ordinalityColumn.getName().getCanonicalValue(); + if (!uniqueNames.add(name)) { + throw semanticException(DUPLICATE_COLUMN_OR_PATH_NAME, ordinalityColumn.getName(), "All column and path names in JSON_TABLE invocation must be unique"); + } + outputFields.add(Field.newUnqualified(name, BIGINT)); + orderedOutputColumns.add(NodeRef.of(ordinalityColumn)); + } + else if (column instanceof ValueColumn valueColumn) { + String name = valueColumn.getName().getCanonicalValue(); + if (!uniqueNames.add(name)) { + throw semanticException(DUPLICATE_COLUMN_OR_PATH_NAME, valueColumn.getName(), "All column and path names in JSON_TABLE invocation must be unique"); + } + valueColumn.getEmptyDefault().ifPresent(expression -> verifyNoAggregateWindowOrGroupingFunctions(session, functionResolver, accessControl, expression, "default expression for JSON_TABLE column")); + valueColumn.getErrorDefault().ifPresent(expression -> verifyNoAggregateWindowOrGroupingFunctions(session, functionResolver, accessControl, expression, "default expression for JSON_TABLE column")); + JsonPathAnalysis pathAnalysis = valueColumn.getJsonPath() + .map(this::analyzeJsonPath) + .orElseGet(() -> analyzeImplicitJsonPath(getImplicitJsonPath(name), valueColumn.getLocation())); + analysis.setJsonPathAnalysis(valueColumn, pathAnalysis); + TypeAndAnalysis typeAndAnalysis = analyzeJsonValueExpression( + valueColumn, + pathAnalysis, + session, + plannerContext, + statementAnalyzerFactory, + accessControl, + enclosingScope, + analysis, + warningCollector, + correlationSupport); + // default values can contain subqueries - the subqueries are recorded under the enclosing JsonTable node + analysis.recordSubqueries(jsonTable, typeAndAnalysis.analysis()); + outputFields.add(Field.newUnqualified(name, typeAndAnalysis.type())); + orderedOutputColumns.add(NodeRef.of(valueColumn)); + } + else if (column instanceof QueryColumn queryColumn) { + String name = queryColumn.getName().getCanonicalValue(); + if (!uniqueNames.add(name)) { + throw semanticException(DUPLICATE_COLUMN_OR_PATH_NAME, queryColumn.getName(), "All column and path names in JSON_TABLE invocation must be unique"); + } + JsonPathAnalysis pathAnalysis = queryColumn.getJsonPath() + .map(this::analyzeJsonPath) + .orElseGet(() -> analyzeImplicitJsonPath(getImplicitJsonPath(name), queryColumn.getLocation())); + analysis.setJsonPathAnalysis(queryColumn, pathAnalysis); + Type type = analyzeJsonQueryExpression(queryColumn, session, plannerContext, statementAnalyzerFactory, accessControl, enclosingScope, analysis, warningCollector); + outputFields.add(Field.newUnqualified(name, type)); + orderedOutputColumns.add(NodeRef.of(queryColumn)); + } + else if (column instanceof NestedColumns nestedColumns) { + nestedColumns.getPathName().ifPresent(name -> { + if (!uniqueNames.add(name.getCanonicalValue())) { + throw semanticException(DUPLICATE_COLUMN_OR_PATH_NAME, name, "All column and path names in JSON_TABLE invocation must be unique"); + } + }); + JsonPathAnalysis pathAnalysis = analyzeJsonPath(nestedColumns.getJsonPath()); + analysis.setJsonPathAnalysis(nestedColumns, pathAnalysis); + analyzeJsonTableColumns(nestedColumns.getColumns(), uniqueNames, outputFields, orderedOutputColumns, enclosingScope, jsonTable); + } + else { + throw new IllegalArgumentException("unexpected type of JSON_TABLE column: " + column.getClass().getSimpleName()); + } + } + } + + private static String getImplicitJsonPath(String name) + { + // TODO the spec misses the path mode. I put 'lax', but it should be confirmed, as the path mode is meaningful for the semantics of the implicit path. + return "lax $.\"" + name.replace("\"", "\"\"") + '"'; + } + + private JsonPathAnalysis analyzeJsonPath(StringLiteral path) + { + return new JsonPathAnalyzer( + plannerContext.getMetadata(), + session, + createConstantAnalyzer(plannerContext, accessControl, session, analysis.getParameters(), WarningCollector.NOOP, analysis.isDescribe())) + .analyzeJsonPath(path, ImmutableMap.of()); + } + + private JsonPathAnalysis analyzeImplicitJsonPath(String path, Optional columnLocation) + { + return new JsonPathAnalyzer( + plannerContext.getMetadata(), + session, + createConstantAnalyzer(plannerContext, accessControl, session, analysis.getParameters(), WarningCollector.NOOP, analysis.isDescribe())) + .analyzeImplicitJsonPath(path, columnLocation.orElseThrow(() -> new IllegalStateException("missing NodeLocation for JSON_TABLE column"))); + } + + private void validateJsonTableSpecificPlan(JsonPathInvocation rootPath, JsonTableSpecificPlan rootPlan, List rootColumns) + { + String rootPathName = rootPath.getPathName() + .orElseThrow(() -> semanticException(MISSING_PATH_NAME, rootPath, "All JSON paths must be named when specific plan is given")) + .getCanonicalValue(); + String rootPlanName; + if (rootPlan instanceof PlanLeaf planLeaf) { + rootPlanName = planLeaf.getName().getCanonicalValue(); + } + else if (rootPlan instanceof PlanParentChild planParentChild) { + rootPlanName = planParentChild.getParent().getName().getCanonicalValue(); + } + else { + throw semanticException(INVALID_PLAN, rootPlan, "JSON_TABLE plan must either be a single path name or it must be rooted in parent-child relationship (OUTER or INNER)"); + } + validateJsonTablePlan(ImmutableMap.of(rootPathName, rootColumns), ImmutableMap.of(rootPlanName, rootPlan), rootPlan); + } + + private void validateJsonTablePlan(Map> actualNodes, Map planNodes, JsonTableSpecificPlan rootPlan) + { + Set unhandledActualNodes = Sets.difference(actualNodes.keySet(), planNodes.keySet()); + if (!unhandledActualNodes.isEmpty()) { + throw semanticException(INVALID_PLAN, rootPlan, "JSON_TABLE plan should contain all JSON paths available at each level of nesting. Paths not included: %s", String.join(", ", unhandledActualNodes)); + } + Set irrelevantPlanChildren = Sets.difference(planNodes.keySet(), actualNodes.keySet()); + if (!irrelevantPlanChildren.isEmpty()) { + throw semanticException(INVALID_PLAN, rootPlan, "JSON_TABLE plan includes unavailable JSON path names: %s", String.join(", ", irrelevantPlanChildren)); + } + + // recurse into child nodes + actualNodes.forEach((name, columns) -> { + JsonTableSpecificPlan plan = planNodes.get(name); + + Map> actualChildren = columns.stream() + .filter(NestedColumns.class::isInstance) + .map(NestedColumns.class::cast) + .collect(toImmutableMap( + child -> child.getPathName() + .orElseThrow(() -> semanticException(MISSING_PATH_NAME, child.getJsonPath(), "All JSON paths must be named when specific plan is given")) + .getCanonicalValue(), + NestedColumns::getColumns)); + + Map planChildren; + if (plan instanceof PlanLeaf) { + planChildren = ImmutableMap.of(); + } + else if (plan instanceof PlanParentChild planParentChild) { + planChildren = new HashMap<>(); + getPlanSiblings(planParentChild.getChild(), planChildren); + } + else { + throw new IllegalStateException("unexpected JSON_TABLE plan node: " + plan.getClass().getSimpleName()); + } + + validateJsonTablePlan(actualChildren, planChildren, rootPlan); + }); + } + + private void getPlanSiblings(JsonTableSpecificPlan plan, Map plansByName) + { + if (plan instanceof PlanLeaf planLeaf) { + if (plansByName.put(planLeaf.getName().getCanonicalValue(), planLeaf) != null) { + throw semanticException(INVALID_PLAN, planLeaf, "Duplicate reference to JSON path name in sibling plan: %s", planLeaf.getName().getCanonicalValue()); + } + } + else if (plan instanceof PlanParentChild planParentChild) { + if (plansByName.put(planParentChild.getParent().getName().getCanonicalValue(), planParentChild) != null) { + throw semanticException(INVALID_PLAN, planParentChild.getParent(), "Duplicate reference to JSON path name in sibling plan: %s", planParentChild.getParent().getName().getCanonicalValue()); + } + } + else if (plan instanceof PlanSiblings planSiblings) { + for (JsonTableSpecificPlan sibling : planSiblings.getSiblings()) { + getPlanSiblings(sibling, plansByName); + } + } + } + + // Per SQL standard ISO/IEC STANDARD 9075-2, p. 453, g), i), and p. 821, 2), b), when PLAN DEFAULT is specified, all nested paths must be named, but the root path does not have to be named. + private void checkAllNestedPathsNamed(List columns) { - throw semanticException(NOT_SUPPORTED, node, "JSON_TABLE is not yet supported"); + List nestedColumns = columns.stream() + .filter(NestedColumns.class::isInstance) + .map(NestedColumns.class::cast) + .collect(toImmutableList()); + + nestedColumns.stream() + .forEach(definition -> { + if (definition.getPathName().isEmpty()) { + throw semanticException(MISSING_PATH_NAME, definition.getJsonPath(), "All nested JSON paths must be named when default plan is given"); + } + }); + + nestedColumns.stream() + .forEach(definition -> checkAllNestedPathsNamed(definition.getColumns())); } private void analyzeWindowDefinitions(QuerySpecification node, Scope scope) diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/RelationPlanner.java b/core/trino-main/src/main/java/io/trino/sql/planner/RelationPlanner.java index a88fa26caf75..cdcc86b8eb3a 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/RelationPlanner.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/RelationPlanner.java @@ -19,14 +19,28 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.ListMultimap; import io.trino.Session; +import io.trino.json.ir.IrJsonPath; +import io.trino.metadata.ResolvedFunction; import io.trino.metadata.TableFunctionHandle; import io.trino.metadata.TableHandle; +import io.trino.operator.table.json.JsonTable.JsonTableFunctionHandle; +import io.trino.operator.table.json.JsonTableColumn; +import io.trino.operator.table.json.JsonTableOrdinalityColumn; +import io.trino.operator.table.json.JsonTablePlanCross; +import io.trino.operator.table.json.JsonTablePlanLeaf; +import io.trino.operator.table.json.JsonTablePlanNode; +import io.trino.operator.table.json.JsonTablePlanSingle; +import io.trino.operator.table.json.JsonTablePlanUnion; +import io.trino.operator.table.json.JsonTableQueryColumn; +import io.trino.operator.table.json.JsonTableValueColumn; import io.trino.spi.connector.ColumnHandle; +import io.trino.spi.function.table.TableArgument; import io.trino.spi.type.RowType; import io.trino.spi.type.Type; import io.trino.sql.ExpressionUtils; import io.trino.sql.PlannerContext; import io.trino.sql.analyzer.Analysis; +import io.trino.sql.analyzer.Analysis.JsonTableAnalysis; import io.trino.sql.analyzer.Analysis.TableArgumentAnalysis; import io.trino.sql.analyzer.Analysis.TableFunctionInvocationAnalysis; import io.trino.sql.analyzer.Analysis.UnnestAnalysis; @@ -34,6 +48,7 @@ import io.trino.sql.analyzer.RelationType; import io.trino.sql.analyzer.Scope; import io.trino.sql.planner.QueryPlanner.PlanAndMappings; +import io.trino.sql.planner.TranslationMap.ParametersRow; import io.trino.sql.planner.plan.Assignments; import io.trino.sql.planner.plan.CorrelatedJoinNode; import io.trino.sql.planner.plan.DataOrganizationSpecification; @@ -61,27 +76,45 @@ import io.trino.sql.planner.rowpattern.ir.IrRowPattern; import io.trino.sql.tree.AliasedRelation; import io.trino.sql.tree.AstVisitor; +import io.trino.sql.tree.BooleanLiteral; import io.trino.sql.tree.Cast; import io.trino.sql.tree.CoalesceExpression; import io.trino.sql.tree.ComparisonExpression; import io.trino.sql.tree.Except; import io.trino.sql.tree.Expression; +import io.trino.sql.tree.FunctionCall; +import io.trino.sql.tree.GenericLiteral; import io.trino.sql.tree.Identifier; import io.trino.sql.tree.IfExpression; import io.trino.sql.tree.Intersect; import io.trino.sql.tree.Join; import io.trino.sql.tree.JoinCriteria; import io.trino.sql.tree.JoinUsing; +import io.trino.sql.tree.JsonPathParameter; +import io.trino.sql.tree.JsonQuery; +import io.trino.sql.tree.JsonTable; +import io.trino.sql.tree.JsonTableColumnDefinition; +import io.trino.sql.tree.JsonTableDefaultPlan; +import io.trino.sql.tree.JsonTablePlan.ParentChildPlanType; +import io.trino.sql.tree.JsonTablePlan.SiblingsPlanType; +import io.trino.sql.tree.JsonTableSpecificPlan; +import io.trino.sql.tree.JsonValue; import io.trino.sql.tree.LambdaArgumentDeclaration; import io.trino.sql.tree.Lateral; import io.trino.sql.tree.MeasureDefinition; import io.trino.sql.tree.NaturalJoin; +import io.trino.sql.tree.NestedColumns; import io.trino.sql.tree.Node; import io.trino.sql.tree.NodeRef; +import io.trino.sql.tree.OrdinalityColumn; import io.trino.sql.tree.PatternRecognitionRelation; import io.trino.sql.tree.PatternSearchMode; +import io.trino.sql.tree.PlanLeaf; +import io.trino.sql.tree.PlanParentChild; +import io.trino.sql.tree.PlanSiblings; import io.trino.sql.tree.QualifiedName; import io.trino.sql.tree.Query; +import io.trino.sql.tree.QueryColumn; import io.trino.sql.tree.QuerySpecification; import io.trino.sql.tree.Relation; import io.trino.sql.tree.Row; @@ -97,6 +130,7 @@ import io.trino.sql.tree.TableSubquery; import io.trino.sql.tree.Union; import io.trino.sql.tree.Unnest; +import io.trino.sql.tree.ValueColumn; import io.trino.sql.tree.Values; import io.trino.sql.tree.VariableDefinition; import io.trino.type.TypeCoercion; @@ -118,6 +152,7 @@ import static io.trino.spi.StandardErrorCode.CONSTRAINT_VIOLATION; import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; import static io.trino.spi.type.BooleanType.BOOLEAN; +import static io.trino.spi.type.StandardTypes.TINYINT; import static io.trino.sql.NodeUtils.getSortItemsFromOrderBy; import static io.trino.sql.analyzer.SemanticExceptions.semanticException; import static io.trino.sql.analyzer.TypeSignatureTranslator.toSqlType; @@ -133,11 +168,18 @@ import static io.trino.sql.planner.plan.AggregationNode.singleGroupingSet; import static io.trino.sql.tree.BooleanLiteral.TRUE_LITERAL; import static io.trino.sql.tree.Join.Type.CROSS; +import static io.trino.sql.tree.Join.Type.FULL; import static io.trino.sql.tree.Join.Type.IMPLICIT; import static io.trino.sql.tree.Join.Type.INNER; +import static io.trino.sql.tree.Join.Type.LEFT; +import static io.trino.sql.tree.JsonQuery.QuotesBehavior.KEEP; +import static io.trino.sql.tree.JsonQuery.QuotesBehavior.OMIT; +import static io.trino.sql.tree.JsonTablePlan.ParentChildPlanType.OUTER; +import static io.trino.sql.tree.JsonTablePlan.SiblingsPlanType.UNION; import static io.trino.sql.tree.PatternRecognitionRelation.RowsPerMatch.ONE; import static io.trino.sql.tree.PatternSearchMode.Mode.INITIAL; import static io.trino.sql.tree.SkipTo.Position.PAST_LAST; +import static io.trino.type.Json2016Type.JSON_2016; import static java.lang.Boolean.TRUE; import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; @@ -685,6 +727,16 @@ protected RelationPlan visitJoin(Join node, Void context) return planJoinUnnest(leftPlan, node, unnest.get()); } + Optional jsonTable = getJsonTable(node.getRight()); + if (jsonTable.isPresent()) { + return planJoinJsonTable( + newPlanBuilder(leftPlan, analysis, lambdaDeclarationToSymbolMap, session, plannerContext), + leftPlan.getFieldMappings(), + node.getType(), + jsonTable.get(), + analysis.getScope(node)); + } + Optional lateral = getLateral(node.getRight()); if (lateral.isPresent()) { return planCorrelatedJoin(node, leftPlan, lateral.get()); @@ -1002,6 +1054,17 @@ private static Optional getUnnest(Relation relation) return Optional.empty(); } + private static Optional getJsonTable(Relation relation) + { + if (relation instanceof AliasedRelation) { + return getJsonTable(((AliasedRelation) relation).getRelation()); + } + if (relation instanceof JsonTable) { + return Optional.of((JsonTable) relation); + } + return Optional.empty(); + } + private static Optional getLateral(Relation relation) { if (relation instanceof AliasedRelation) { @@ -1124,6 +1187,393 @@ private RelationPlan planUnnest(PlanBuilder subPlan, Unnest node, List r return new RelationPlan(unnestNode, outputScope, unnestNode.getOutputSymbols(), outerContext); } + private RelationPlan planJoinJsonTable(PlanBuilder leftPlan, List leftFieldMappings, Join.Type joinType, JsonTable jsonTable, Scope outputScope) + { + PlanBuilder planBuilder = leftPlan; + + // extract input expressions + ImmutableList.Builder builder = ImmutableList.builder(); + Expression inputExpression = jsonTable.getJsonPathInvocation().getInputExpression(); + builder.add(inputExpression); + List pathParameters = jsonTable.getJsonPathInvocation().getPathParameters(); + pathParameters.stream() + .map(JsonPathParameter::getParameter) + .forEach(builder::add); + List defaultExpressions = getDefaultExpressions(jsonTable.getColumns()); + builder.addAll(defaultExpressions); + List inputExpressions = builder.build(); + + planBuilder = subqueryPlanner.handleSubqueries(planBuilder, inputExpressions, analysis.getSubqueries(jsonTable)); + planBuilder = planBuilder.appendProjections(inputExpressions, symbolAllocator, idAllocator); + + // apply coercions + // coercions might be necessary for the context item and path parameters before the input functions are applied + // also, the default expressions in value columns (DEFAULT ... ON EMPTY / ON ERROR) might need a coercion to match the required output type + PlanAndMappings coerced = coerce(planBuilder, inputExpressions, analysis, idAllocator, symbolAllocator, typeCoercion); + planBuilder = coerced.getSubPlan(); + + // apply the input function to the input expression + BooleanLiteral failOnError = new BooleanLiteral(jsonTable.getErrorBehavior().orElse(JsonTable.ErrorBehavior.EMPTY) == JsonTable.ErrorBehavior.ERROR ? "true" : "false"); + ResolvedFunction inputToJson = analysis.getJsonInputFunction(inputExpression); + Expression inputJson = new FunctionCall(inputToJson.toQualifiedName(), ImmutableList.of(coerced.get(inputExpression).toSymbolReference(), failOnError)); + + // apply the input functions to the JSON path parameters having FORMAT, + // and collect all JSON path parameters in a Row + List coercedParameters = pathParameters.stream() + .map(parameter -> new JsonPathParameter( + parameter.getLocation(), + parameter.getName(), + coerced.get(parameter.getParameter()).toSymbolReference(), + parameter.getFormat())) + .collect(toImmutableList()); + JsonTableAnalysis jsonTableAnalysis = analysis.getJsonTableAnalysis(jsonTable); + RowType parametersType = jsonTableAnalysis.parametersType(); + ParametersRow orderedParameters = planBuilder.getTranslations().getParametersRow(pathParameters, coercedParameters, parametersType, failOnError); + Expression parametersRow = orderedParameters.getParametersRow(); + + // append projections for inputJson and parametersRow + // cannot use the 'appendProjections()' method because the projected expressions include resolved input functions, so they are not pure AST expressions + Symbol inputJsonSymbol = symbolAllocator.newSymbol("inputJson", JSON_2016); + Symbol parametersRowSymbol = symbolAllocator.newSymbol("parametersRow", parametersType); + ProjectNode appended = new ProjectNode( + idAllocator.getNextId(), + planBuilder.getRoot(), + Assignments.builder() + .putIdentities(planBuilder.getRoot().getOutputSymbols()) + .put(inputJsonSymbol, inputJson) + .put(parametersRowSymbol, parametersRow) + .build()); + planBuilder = planBuilder.withNewRoot(appended); + + // identify the required symbols + ImmutableList.Builder requiredSymbolsBuilder = ImmutableList.builder() + .add(inputJsonSymbol) + .add(parametersRowSymbol); + defaultExpressions.stream() + .map(coerced::get) + .distinct() + .forEach(requiredSymbolsBuilder::add); + List requiredSymbols = requiredSymbolsBuilder.build(); + + // map the default expressions of value columns to indexes in the required columns list + // use a HashMap because there might be duplicate expressions + Map defaultExpressionsMapping = new HashMap<>(); + for (Expression defaultExpression : defaultExpressions) { + defaultExpressionsMapping.put(defaultExpression, requiredSymbols.indexOf(coerced.get(defaultExpression))); + } + + // rewrite the root JSON path to IR using parameters + IrJsonPath rootPath = new JsonPathTranslator(session, plannerContext).rewriteToIr(analysis.getJsonPathAnalysis(jsonTable), orderedParameters.getParametersOrder()); + + // create json_table execution plan + List> orderedColumns = jsonTableAnalysis.orderedOutputColumns(); + Map, Integer> outputIndexMapping = IntStream.range(0, orderedColumns.size()) + .boxed() + .collect(toImmutableMap(orderedColumns::get, Function.identity())); + JsonTablePlanNode executionPlan; + boolean defaultErrorOnError = jsonTable.getErrorBehavior().map(errorBehavior -> errorBehavior == JsonTable.ErrorBehavior.ERROR).orElse(false); + if (jsonTable.getPlan().isEmpty()) { + executionPlan = getPlanFromDefaults(rootPath, jsonTable.getColumns(), OUTER, UNION, defaultErrorOnError, outputIndexMapping, defaultExpressionsMapping); + } + else if (jsonTable.getPlan().orElseThrow() instanceof JsonTableDefaultPlan defaultPlan) { + executionPlan = getPlanFromDefaults(rootPath, jsonTable.getColumns(), defaultPlan.getParentChild(), defaultPlan.getSiblings(), defaultErrorOnError, outputIndexMapping, defaultExpressionsMapping); + } + else { + executionPlan = getPlanFromSpecification(rootPath, jsonTable.getColumns(), (JsonTableSpecificPlan) jsonTable.getPlan().orElseThrow(), defaultErrorOnError, outputIndexMapping, defaultExpressionsMapping); + } + + // create new symbols for json_table function's proper columns + // These are the types produced by the table function. + // For ordinality and value columns, the types match the expected output type. + // Query columns return JSON_2016. Later we need to apply an output function, and potentially a coercion to match the declared output type. + RelationType jsonTableRelationType = analysis.getScope(jsonTable).getRelationType(); + List properOutputs = IntStream.range(0, orderedColumns.size()) + .mapToObj(index -> { + if (orderedColumns.get(index).getNode() instanceof QueryColumn queryColumn) { + return symbolAllocator.newSymbol(queryColumn.getName().getCanonicalValue(), JSON_2016); + } + return symbolAllocator.newSymbol(jsonTableRelationType.getFieldByIndex(index)); + }) + .collect(toImmutableList()); + + // pass through all columns from the left side of the join + List passThroughColumns = leftFieldMappings.stream() + .map(symbol -> new PassThroughColumn(symbol, false)) + .collect(toImmutableList()); + + // determine the join type between the input, and the json_table result + // this join type is not described in the plan, it depends on the enclosing join whose right source is the json_table + // since json_table is a lateral relation, and the join condition is 'true', effectively the join type is either LEFT OUTER or INNER + boolean outer = joinType == LEFT || joinType == FULL; + + // create the TableFunctionNode and TableFunctionHandle + JsonTableFunctionHandle functionHandle = new JsonTableFunctionHandle( + executionPlan, + outer, + defaultErrorOnError, + parametersType, + properOutputs.stream() + .map(symbolAllocator.getTypes()::get) + .toArray(Type[]::new)); + + TableFunctionNode tableFunctionNode = new TableFunctionNode( + idAllocator.getNextId(), + "$json_table", + jsonTableAnalysis.catalogHandle(), + ImmutableMap.of("$input", new TableArgument(getRowType(planBuilder.getRoot()), ImmutableList.of(), ImmutableList.of())), + properOutputs, + ImmutableList.of(planBuilder.getRoot()), + ImmutableList.of(new TableArgumentProperties( + "$input", + true, + true, + new PassThroughSpecification(true, passThroughColumns), + requiredSymbols, + Optional.empty())), + ImmutableList.of(), + new TableFunctionHandle( + jsonTableAnalysis.catalogHandle(), + functionHandle, + jsonTableAnalysis.transactionHandle())); + + // append output functions and coercions for query columns + // The table function returns JSON_2016 for query columns. We need to apply output functions and coercions to match the declared output type. + // create output layout: first the left side of the join, next the proper columns + ImmutableList.Builder outputLayout = ImmutableList.builder() + .addAll(leftFieldMappings); + Assignments.Builder assignments = Assignments.builder() + .putIdentities(leftFieldMappings); + for (int i = 0; i < properOutputs.size(); i++) { + Symbol properOutput = properOutputs.get(i); + if (orderedColumns.get(i).getNode() instanceof QueryColumn queryColumn) { + // apply output function + GenericLiteral errorBehavior = new GenericLiteral( + TINYINT, + String.valueOf(queryColumn.getErrorBehavior().orElse(defaultErrorOnError ? JsonQuery.EmptyOrErrorBehavior.ERROR : JsonQuery.EmptyOrErrorBehavior.NULL).ordinal())); + BooleanLiteral omitQuotes = new BooleanLiteral(queryColumn.getQuotesBehavior().orElse(KEEP) == OMIT ? "true" : "false"); + ResolvedFunction outputFunction = analysis.getJsonOutputFunction(queryColumn); + Expression result = new FunctionCall(outputFunction.toQualifiedName(), ImmutableList.of(properOutput.toSymbolReference(), errorBehavior, omitQuotes)); + + // cast to declared returned type + Type expectedType = jsonTableRelationType.getFieldByIndex(i).getType(); + Type resultType = outputFunction.getSignature().getReturnType(); + if (!resultType.equals(expectedType)) { + result = new Cast(result, toSqlType(expectedType)); + } + + Symbol output = symbolAllocator.newSymbol(result, expectedType); + outputLayout.add(output); + assignments.put(output, result); + } + else { + outputLayout.add(properOutput); + assignments.putIdentity(properOutput); + } + } + + ProjectNode projectNode = new ProjectNode( + idAllocator.getNextId(), + tableFunctionNode, + assignments.build()); + + return new RelationPlan(projectNode, outputScope, outputLayout.build(), outerContext); + } + + private static List getDefaultExpressions(List columns) + { + ImmutableList.Builder builder = ImmutableList.builder(); + for (JsonTableColumnDefinition column : columns) { + if (column instanceof ValueColumn valueColumn) { + valueColumn.getEmptyDefault().ifPresent(builder::add); + valueColumn.getErrorDefault().ifPresent(builder::add); + } + else if (column instanceof NestedColumns nestedColumns) { + builder.addAll(getDefaultExpressions(nestedColumns.getColumns())); + } + } + return builder.build(); + } + + private JsonTablePlanNode getPlanFromDefaults( + IrJsonPath path, + List columnDefinitions, + ParentChildPlanType parentChildPlanType, + SiblingsPlanType siblingsPlanType, + boolean defaultErrorOnError, + Map, Integer> outputIndexMapping, + Map defaultExpressionsMapping) + { + ImmutableList.Builder columns = ImmutableList.builder(); + ImmutableList.Builder childrenBuilder = ImmutableList.builder(); + + for (JsonTableColumnDefinition columnDefinition : columnDefinitions) { + if (columnDefinition instanceof NestedColumns nestedColumns) { + IrJsonPath nestedPath = new JsonPathTranslator(session, plannerContext).rewriteToIr(analysis.getJsonPathAnalysis(nestedColumns), ImmutableList.of()); + childrenBuilder.add(getPlanFromDefaults( + nestedPath, + nestedColumns.getColumns(), + parentChildPlanType, + siblingsPlanType, + defaultErrorOnError, + outputIndexMapping, + defaultExpressionsMapping)); + } + else { + columns.add(getColumn(columnDefinition, defaultErrorOnError, outputIndexMapping, defaultExpressionsMapping)); + } + } + + List children = childrenBuilder.build(); + if (children.isEmpty()) { + return new JsonTablePlanLeaf(path, columns.build()); + } + + JsonTablePlanNode child; + if (children.size() == 1) { + child = getOnlyElement(children); + } + else if (siblingsPlanType == UNION) { + child = new JsonTablePlanUnion(children); + } + else { + child = new JsonTablePlanCross(children); + } + + return new JsonTablePlanSingle(path, columns.build(), parentChildPlanType == OUTER, child); + } + + private JsonTablePlanNode getPlanFromSpecification( + IrJsonPath path, + List columnDefinitions, + JsonTableSpecificPlan specificPlan, + boolean defaultErrorOnError, + Map, Integer> outputIndexMapping, + Map defaultExpressionsMapping) + { + ImmutableList.Builder columns = ImmutableList.builder(); + ImmutableMap.Builder childrenBuilder = ImmutableMap.builder(); + Map planSiblings; + if (specificPlan instanceof PlanLeaf) { + planSiblings = ImmutableMap.of(); + } + else { + planSiblings = getSiblings(((PlanParentChild) specificPlan).getChild()); + } + + for (JsonTableColumnDefinition columnDefinition : columnDefinitions) { + if (columnDefinition instanceof NestedColumns nestedColumns) { + IrJsonPath nestedPath = new JsonPathTranslator(session, plannerContext).rewriteToIr(analysis.getJsonPathAnalysis(nestedColumns), ImmutableList.of()); + String nestedPathName = nestedColumns.getPathName().orElseThrow().getCanonicalValue(); + JsonTablePlanNode child = getPlanFromSpecification( + nestedPath, + nestedColumns.getColumns(), + planSiblings.get(nestedPathName), + defaultErrorOnError, + outputIndexMapping, + defaultExpressionsMapping); + childrenBuilder.put(nestedPathName, child); + } + else { + columns.add(getColumn(columnDefinition, defaultErrorOnError, outputIndexMapping, defaultExpressionsMapping)); + } + } + + Map children = childrenBuilder.buildOrThrow(); + if (children.isEmpty()) { + return new JsonTablePlanLeaf(path, columns.build()); + } + + PlanParentChild planParentChild = (PlanParentChild) specificPlan; + boolean outer = planParentChild.getType() == OUTER; + JsonTablePlanNode child = combineSiblings(children, planParentChild.getChild()); + return new JsonTablePlanSingle(path, columns.build(), outer, child); + } + + private Map getSiblings(JsonTableSpecificPlan plan) + { + if (plan instanceof PlanLeaf planLeaf) { + return ImmutableMap.of(planLeaf.getName().getCanonicalValue(), planLeaf); + } + if (plan instanceof PlanParentChild planParentChild) { + return ImmutableMap.of(planParentChild.getParent().getName().getCanonicalValue(), planParentChild); + } + PlanSiblings planSiblings = (PlanSiblings) plan; + ImmutableMap.Builder siblings = ImmutableMap.builder(); + for (JsonTableSpecificPlan sibling : planSiblings.getSiblings()) { + siblings.putAll(getSiblings(sibling)); + } + return siblings.buildOrThrow(); + } + + private JsonTableColumn getColumn( + JsonTableColumnDefinition columnDefinition, + boolean defaultErrorOnError, + Map, Integer> outputIndexMapping, + Map defaultExpressionsMapping) + { + int index = outputIndexMapping.get(NodeRef.of(columnDefinition)); + + if (columnDefinition instanceof OrdinalityColumn) { + return new JsonTableOrdinalityColumn(index); + } + ResolvedFunction columnFunction = analysis.getResolvedFunction(columnDefinition); + IrJsonPath columnPath = new JsonPathTranslator(session, plannerContext).rewriteToIr(analysis.getJsonPathAnalysis(columnDefinition), ImmutableList.of()); + if (columnDefinition instanceof QueryColumn queryColumn) { + return new JsonTableQueryColumn( + index, + columnFunction, + columnPath, + queryColumn.getWrapperBehavior().ordinal(), + queryColumn.getEmptyBehavior().ordinal(), + queryColumn.getErrorBehavior().orElse(defaultErrorOnError ? JsonQuery.EmptyOrErrorBehavior.ERROR : JsonQuery.EmptyOrErrorBehavior.NULL).ordinal()); + } + if (columnDefinition instanceof ValueColumn valueColumn) { + int emptyDefault = valueColumn.getEmptyDefault() + .map(defaultExpressionsMapping::get) + .orElse(-1); + int errorDefault = valueColumn.getErrorDefault() + .map(defaultExpressionsMapping::get) + .orElse(-1); + return new JsonTableValueColumn( + index, + columnFunction, + columnPath, + valueColumn.getEmptyBehavior().ordinal(), + emptyDefault, + valueColumn.getErrorBehavior().orElse(defaultErrorOnError ? JsonValue.EmptyOrErrorBehavior.ERROR : JsonValue.EmptyOrErrorBehavior.NULL).ordinal(), + errorDefault); + } + throw new IllegalStateException("unexpected column definition: " + columnDefinition.getClass().getSimpleName()); + } + + private JsonTablePlanNode combineSiblings(Map siblings, JsonTableSpecificPlan plan) + { + if (plan instanceof PlanLeaf planLeaf) { + return siblings.get(planLeaf.getName().getCanonicalValue()); + } + if (plan instanceof PlanParentChild planParentChild) { + return siblings.get(planParentChild.getParent().getName().getCanonicalValue()); + } + PlanSiblings planSiblings = (PlanSiblings) plan; + List siblingNodes = planSiblings.getSiblings().stream() + .map(sibling -> combineSiblings(siblings, sibling)) + .collect(toImmutableList()); + if (planSiblings.getType() == UNION) { + return new JsonTablePlanUnion(siblingNodes); + } + return new JsonTablePlanCross(siblingNodes); + } + + private RowType getRowType(PlanNode node) + { + // create a RowType based on output symbols of a node + // The node is an intermediate stage of planning json_table. There's no recorded relation type available for this node. + // The returned RowType is only used in plan printer + return RowType.from(node.getOutputSymbols().stream() + .map(symbol -> new RowType.Field(Optional.of(symbol.getName()), symbolAllocator.getTypes().get(symbol))) + .collect(toImmutableList())); + } + @Override protected RelationPlan visitTableSubquery(TableSubquery node, Void context) { @@ -1206,6 +1656,17 @@ private PlanBuilder planSingleEmptyRow(Optional parent) return new PlanBuilder(translations, values); } + @Override + protected RelationPlan visitJsonTable(JsonTable node, Void context) + { + return planJoinJsonTable( + planSingleEmptyRow(analysis.getScope(node).getOuterQueryParent()), + ImmutableList.of(), + INNER, + node, + analysis.getScope(node)); + } + @Override protected RelationPlan visitUnion(Union node, Void context) { diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/ResolvedFunctionCallRewriter.java b/core/trino-main/src/main/java/io/trino/sql/planner/ResolvedFunctionCallRewriter.java index 2a5e457e611a..8b2c6c985106 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/ResolvedFunctionCallRewriter.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/ResolvedFunctionCallRewriter.java @@ -18,6 +18,7 @@ import io.trino.sql.tree.ExpressionRewriter; import io.trino.sql.tree.ExpressionTreeRewriter; import io.trino.sql.tree.FunctionCall; +import io.trino.sql.tree.Node; import io.trino.sql.tree.NodeRef; import java.util.Map; @@ -29,7 +30,7 @@ public final class ResolvedFunctionCallRewriter { private ResolvedFunctionCallRewriter() {} - public static Expression rewriteResolvedFunctions(Expression expression, Map, ResolvedFunction> resolvedFunctions) + public static Expression rewriteResolvedFunctions(Expression expression, Map, ResolvedFunction> resolvedFunctions) { return ExpressionTreeRewriter.rewriteWith(new Visitor(resolvedFunctions), expression); } @@ -37,9 +38,9 @@ public static Expression rewriteResolvedFunctions(Expression expression, Map { - private final Map, ResolvedFunction> resolvedFunctions; + private final Map, ResolvedFunction> resolvedFunctions; - public Visitor(Map, ResolvedFunction> resolvedFunctions) + public Visitor(Map, ResolvedFunction> resolvedFunctions) { this.resolvedFunctions = requireNonNull(resolvedFunctions, "resolvedFunctions is null"); } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/TranslationMap.java b/core/trino-main/src/main/java/io/trino/sql/planner/TranslationMap.java index 9e5ec42b8aa4..29de734e8804 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/TranslationMap.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/TranslationMap.java @@ -913,40 +913,6 @@ public Expression rewriteJsonQuery(JsonQuery node, Void context, ExpressionTreeR return coerceIfNecessary(node, result); } - private ParametersRow getParametersRow( - List pathParameters, - List rewrittenPathParameters, - Type parameterRowType, - BooleanLiteral failOnError) - { - Expression parametersRow; - List parametersOrder; - if (!pathParameters.isEmpty()) { - ImmutableList.Builder parameters = ImmutableList.builder(); - for (int i = 0; i < pathParameters.size(); i++) { - ResolvedFunction parameterToJson = analysis.getJsonInputFunction(pathParameters.get(i).getParameter()); - Expression rewrittenParameter = rewrittenPathParameters.get(i).getParameter(); - if (parameterToJson != null) { - parameters.add(new FunctionCall(parameterToJson.toQualifiedName(), ImmutableList.of(rewrittenParameter, failOnError))); - } - else { - parameters.add(rewrittenParameter); - } - } - parametersRow = new Cast(new Row(parameters.build()), toSqlType(parameterRowType)); - parametersOrder = pathParameters.stream() - .map(parameter -> parameter.getName().getCanonicalValue()) - .collect(toImmutableList()); - } - else { - checkState(JSON_NO_PARAMETERS_ROW_TYPE.equals(parameterRowType), "invalid type of parameters row when no parameters are passed"); - parametersRow = new Cast(new NullLiteral(), toSqlType(JSON_NO_PARAMETERS_ROW_TYPE)); - parametersOrder = ImmutableList.of(); - } - - return new ParametersRow(parametersRow, parametersOrder); - } - @Override public Expression rewriteJsonObject(JsonObject node, Void context, ExpressionTreeRewriter treeRewriter) { @@ -1132,7 +1098,41 @@ public Scope getScope() return scope; } - private static class ParametersRow + public ParametersRow getParametersRow( + List pathParameters, + List rewrittenPathParameters, + Type parameterRowType, + BooleanLiteral failOnError) + { + Expression parametersRow; + List parametersOrder; + if (!pathParameters.isEmpty()) { + ImmutableList.Builder parameters = ImmutableList.builder(); + for (int i = 0; i < pathParameters.size(); i++) { + ResolvedFunction parameterToJson = analysis.getJsonInputFunction(pathParameters.get(i).getParameter()); + Expression rewrittenParameter = rewrittenPathParameters.get(i).getParameter(); + if (parameterToJson != null) { + parameters.add(new FunctionCall(parameterToJson.toQualifiedName(), ImmutableList.of(rewrittenParameter, failOnError))); + } + else { + parameters.add(rewrittenParameter); + } + } + parametersRow = new Cast(new Row(parameters.build()), toSqlType(parameterRowType)); + parametersOrder = pathParameters.stream() + .map(parameter -> parameter.getName().getCanonicalValue()) + .collect(toImmutableList()); + } + else { + checkState(JSON_NO_PARAMETERS_ROW_TYPE.equals(parameterRowType), "invalid type of parameters row when no parameters are passed"); + parametersRow = new Cast(new NullLiteral(), toSqlType(JSON_NO_PARAMETERS_ROW_TYPE)); + parametersOrder = ImmutableList.of(); + } + + return new ParametersRow(parametersRow, parametersOrder); + } + + public static class ParametersRow { private final Expression parametersRow; private final List parametersOrder; diff --git a/core/trino-main/src/main/java/io/trino/testing/LocalQueryRunner.java b/core/trino-main/src/main/java/io/trino/testing/LocalQueryRunner.java index e922227cca36..36c2514e6eb5 100644 --- a/core/trino-main/src/main/java/io/trino/testing/LocalQueryRunner.java +++ b/core/trino-main/src/main/java/io/trino/testing/LocalQueryRunner.java @@ -378,7 +378,10 @@ private LocalQueryRunner( TypeManager typeManager = new InternalTypeManager(typeRegistry); InternalBlockEncodingSerde blockEncodingSerde = new InternalBlockEncodingSerde(blockEncodingManager, typeManager); - this.globalFunctionCatalog = new GlobalFunctionCatalog(); + this.globalFunctionCatalog = new GlobalFunctionCatalog( + this::getMetadata, + this::getTypeManager, + this::getFunctionManager); globalFunctionCatalog.addFunctions(new InternalFunctionBundle(new LiteralFunction(blockEncodingSerde))); globalFunctionCatalog.addFunctions(SystemFunctionBundle.create(featuresConfig, typeOperators, blockTypeOperators, nodeManager.getCurrentNode().getNodeVersion())); this.groupProvider = new TestingGroupProviderManager(); diff --git a/core/trino-main/src/test/java/io/trino/dispatcher/TestLocalDispatchQuery.java b/core/trino-main/src/test/java/io/trino/dispatcher/TestLocalDispatchQuery.java index d74a931f7b1e..3a0b59b5b195 100644 --- a/core/trino-main/src/test/java/io/trino/dispatcher/TestLocalDispatchQuery.java +++ b/core/trino-main/src/test/java/io/trino/dispatcher/TestLocalDispatchQuery.java @@ -133,7 +133,10 @@ public void testSubmittedForDispatchedQuery() metadata, new FunctionManager( new ConnectorCatalogServiceProvider<>("function provider", new NoConnectorServicesProvider(), ConnectorServices::getFunctionProvider), - new GlobalFunctionCatalog(), + new GlobalFunctionCatalog( + () -> { throw new UnsupportedOperationException(); }, + () -> { throw new UnsupportedOperationException(); }, + () -> { throw new UnsupportedOperationException(); }), LanguageFunctionProvider.DISABLED), new QueryMonitorConfig()); CreateTable createTable = new CreateTable(QualifiedName.of("table"), ImmutableList.of(), FAIL, ImmutableList.of(), Optional.empty()); diff --git a/core/trino-main/src/test/java/io/trino/metadata/TestGlobalFunctionCatalog.java b/core/trino-main/src/test/java/io/trino/metadata/TestGlobalFunctionCatalog.java index 8c756bd8da6d..2c6a42d9e975 100644 --- a/core/trino-main/src/test/java/io/trino/metadata/TestGlobalFunctionCatalog.java +++ b/core/trino-main/src/test/java/io/trino/metadata/TestGlobalFunctionCatalog.java @@ -100,7 +100,10 @@ public void testDuplicateFunctions() FunctionBundle functionBundle = extractFunctions(CustomAdd.class); TypeOperators typeOperators = new TypeOperators(); - GlobalFunctionCatalog globalFunctionCatalog = new GlobalFunctionCatalog(); + GlobalFunctionCatalog globalFunctionCatalog = new GlobalFunctionCatalog( + () -> { throw new UnsupportedOperationException(); }, + () -> { throw new UnsupportedOperationException(); }, + () -> { throw new UnsupportedOperationException(); }); globalFunctionCatalog.addFunctions(SystemFunctionBundle.create(new FeaturesConfig(), typeOperators, new BlockTypeOperators(typeOperators), NodeVersion.UNKNOWN)); globalFunctionCatalog.addFunctions(functionBundle); assertThatThrownBy(() -> globalFunctionCatalog.addFunctions(functionBundle)) @@ -114,7 +117,10 @@ public void testConflictingScalarAggregation() FunctionBundle functions = extractFunctions(ScalarSum.class); TypeOperators typeOperators = new TypeOperators(); - GlobalFunctionCatalog globalFunctionCatalog = new GlobalFunctionCatalog(); + GlobalFunctionCatalog globalFunctionCatalog = new GlobalFunctionCatalog( + () -> { throw new UnsupportedOperationException(); }, + () -> { throw new UnsupportedOperationException(); }, + () -> { throw new UnsupportedOperationException(); }); globalFunctionCatalog.addFunctions(SystemFunctionBundle.create(new FeaturesConfig(), typeOperators, new BlockTypeOperators(typeOperators), NodeVersion.UNKNOWN)); assertThatThrownBy(() -> globalFunctionCatalog.addFunctions(functions)) .isInstanceOf(IllegalStateException.class) diff --git a/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java b/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java index 613eb7ebbdf3..8478d856e9c2 100644 --- a/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java +++ b/core/trino-main/src/test/java/io/trino/sql/analyzer/TestAnalyzer.java @@ -118,6 +118,7 @@ import static io.trino.spi.StandardErrorCode.COLUMN_NOT_FOUND; import static io.trino.spi.StandardErrorCode.COLUMN_TYPE_UNKNOWN; import static io.trino.spi.StandardErrorCode.DUPLICATE_COLUMN_NAME; +import static io.trino.spi.StandardErrorCode.DUPLICATE_COLUMN_OR_PATH_NAME; import static io.trino.spi.StandardErrorCode.DUPLICATE_NAMED_QUERY; import static io.trino.spi.StandardErrorCode.DUPLICATE_PARAMETER_NAME; import static io.trino.spi.StandardErrorCode.DUPLICATE_PROPERTY; @@ -143,6 +144,7 @@ import static io.trino.spi.StandardErrorCode.INVALID_PARTITION_BY; import static io.trino.spi.StandardErrorCode.INVALID_PATH; import static io.trino.spi.StandardErrorCode.INVALID_PATTERN_RECOGNITION_FUNCTION; +import static io.trino.spi.StandardErrorCode.INVALID_PLAN; import static io.trino.spi.StandardErrorCode.INVALID_PROCESSING_MODE; import static io.trino.spi.StandardErrorCode.INVALID_RANGE; import static io.trino.spi.StandardErrorCode.INVALID_RECURSIVE_REFERENCE; @@ -160,6 +162,7 @@ import static io.trino.spi.StandardErrorCode.MISSING_GROUP_BY; import static io.trino.spi.StandardErrorCode.MISSING_ORDER_BY; import static io.trino.spi.StandardErrorCode.MISSING_OVER; +import static io.trino.spi.StandardErrorCode.MISSING_PATH_NAME; import static io.trino.spi.StandardErrorCode.MISSING_ROW_PATTERN; import static io.trino.spi.StandardErrorCode.MISSING_SCHEMA_NAME; import static io.trino.spi.StandardErrorCode.MISSING_VARIABLE_DEFINITIONS; @@ -6738,11 +6741,541 @@ public void testTableFunctionRequiredColumns() } @Test - public void testJsonTable() + public void testJsonTableColumnTypes() { - assertFails("SELECT * FROM JSON_TABLE('[1, 2, 3]', 'lax $[2]' COLUMNS(o FOR ORDINALITY))") + // ordinality column + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS( + o FOR ORDINALITY)) + """); + + // regular column + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' + COLUMNS( + id BIGINT + PATH 'lax $[1]' + DEFAULT 0 ON EMPTY + ERROR ON ERROR)) + """); + + // formatted column + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' + COLUMNS( + id VARBINARY + FORMAT JSON ENCODING UTF16 + PATH 'lax $[1]' + WITHOUT WRAPPER + OMIT QUOTES + EMPTY ARRAY ON EMPTY + NULL ON ERROR)) + """); + + // nested columns + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' + COLUMNS( + NESTED PATH 'lax $[*]' AS nested_path COLUMNS ( + o FOR ORDINALITY, + id BIGINT PATH 'lax $[1]'))) + """); + } + + @Test + public void testJsonTableColumnAndPathNameUniqueness() + { + // root path is named + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' AS root_path + COLUMNS( + o FOR ORDINALITY)) + """); + + // nested path is named + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' + COLUMNS( + NESTED PATH 'lax $[*]' AS nested_path COLUMNS ( + o FOR ORDINALITY))) + """); + + // root and nested paths are named + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + NESTED PATH 'lax $[*]' AS nested_path COLUMNS ( + o FOR ORDINALITY))) + """); + + // duplicate path name + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS some_path + COLUMNS( + NESTED PATH 'lax $[*]' AS some_path COLUMNS ( + o FOR ORDINALITY))) + """) + .hasErrorCode(DUPLICATE_COLUMN_OR_PATH_NAME) + .hasMessage("line 6:35: All column and path names in JSON_TABLE invocation must be unique"); + + // duplicate column name + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS( + id FOR ORDINALITY, + id BIGINT)) + """) + .hasErrorCode(DUPLICATE_COLUMN_OR_PATH_NAME) + .hasMessage("line 7:9: All column and path names in JSON_TABLE invocation must be unique"); + + // column and path names are the same + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' AS some_name + COLUMNS( + some_name FOR ORDINALITY)) + """) + .hasErrorCode(DUPLICATE_COLUMN_OR_PATH_NAME) + .hasMessage("line 6:9: All column and path names in JSON_TABLE invocation must be unique"); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' + COLUMNS( + NESTED PATH 'lax $[*]' AS some_name COLUMNS ( + some_name FOR ORDINALITY))) + """) + .hasErrorCode(DUPLICATE_COLUMN_OR_PATH_NAME) + .hasMessage("line 7:13: All column and path names in JSON_TABLE invocation must be unique"); + + // duplicate name is deeply nested + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS( + NESTED PATH 'lax $[*]' AS some_name COLUMNS ( + NESTED PATH 'lax $' AS another_name COLUMNS ( + NESTED PATH 'lax $' AS yet_another_name COLUMNS ( + some_name FOR ORDINALITY))))) + """) + .hasErrorCode(DUPLICATE_COLUMN_OR_PATH_NAME) + .hasMessage("line 9:21: All column and path names in JSON_TABLE invocation must be unique"); + } + + @Test + public void testJsonTableColumnAndPathNameIdentifierSemantics() + { + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' AS some_name + COLUMNS( + Some_Name FOR ORDINALITY)) + """) + .hasErrorCode(DUPLICATE_COLUMN_OR_PATH_NAME) + .hasMessage("line 6:9: All column and path names in JSON_TABLE invocation must be unique"); + + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' AS some_name + COLUMNS( + "some_name" FOR ORDINALITY)) + """); + } + + @Test + public void testJsonTableOutputColumns() + { + analyze(""" + SELECT a, b, c, d, e + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' + COLUMNS( + a FOR ORDINALITY, + b BIGINT, + c VARBINARY FORMAT JSON ENCODING UTF16, + NESTED PATH 'lax $[*]' COLUMNS ( + d FOR ORDINALITY, + e BIGINT))) + """); + } + + @Test + public void testImplicitJsonPath() + { + // column name: Ab + // canonical name: AB + // implicit path: lax $."AB" + // resolved member accessor: $.AB + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS(Ab BIGINT)) + """); + + // column name: Ab + // canonical name: Ab + // implicit path: lax $."Ab" + // resolved member accessor: $.Ab + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS("Ab" BIGINT)) + """); + + // column name: ? + // canonical name: ? + // implicit path: lax $."?" + // resolved member accessor: $.? + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS("?" BIGINT)) + """); + + // column name: " + // canonical name: " + // implicit path: lax $."""" + // resolved member accessor $." + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS("\"\"" BIGINT)) + """); + } + + @Test + public void testJsonTableSpecificPlan() + { + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS(id BIGINT) + PLAN (root_path)) + """) + .hasErrorCode(MISSING_PATH_NAME) + .hasMessage("line 3:5: All JSON paths must be named when specific plan is given"); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' AS root_path + COLUMNS(id BIGINT) + PLAN (root_path UNION another_path)) + """) + .hasErrorCode(INVALID_PLAN) + .hasMessage("line 6:11: JSON_TABLE plan must either be a single path name or it must be rooted in parent-child relationship (OUTER or INNER)"); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS(id BIGINT) + PLAN (another_path)) + """) + .hasErrorCode(INVALID_PLAN) + .hasMessage("line 6:11: JSON_TABLE plan should contain all JSON paths available at each level of nesting. Paths not included: ROOT_PATH"); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + NESTED PATH 'lax $' COLUMNS(id BIGINT)) + PLAN (root_path OUTER another_path)) + """) + .hasErrorCode(MISSING_PATH_NAME) + .hasMessage("line 6:21: All JSON paths must be named when specific plan is given"); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + NESTED PATH 'lax $' AS nested_path_1 COLUMNS(id_1 BIGINT), + NESTED PATH 'lax $' AS nested_path_2 COLUMNS(id_2 BIGINT)) + PLAN (root_path OUTER (nested_path_1 CROSS another_path))) + """) + .hasErrorCode(INVALID_PLAN) + .hasMessage("line 8:11: JSON_TABLE plan should contain all JSON paths available at each level of nesting. Paths not included: NESTED_PATH_2"); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + NESTED PATH 'lax $' AS nested_path_1 COLUMNS(id_1 BIGINT), + NESTED PATH 'lax $' AS nested_path_2 COLUMNS(id_2 BIGINT)) + PLAN (root_path OUTER (nested_path_1 CROSS another_path CROSS nested_path_2))) + """) + .hasErrorCode(INVALID_PLAN) + .hasMessage("line 8:11: JSON_TABLE plan includes unavailable JSON path names: ANOTHER_PATH"); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + NESTED PATH 'lax $' AS nested_path_1 COLUMNS(id_1 BIGINT), + NESTED PATH 'lax $' AS nested_path_2 COLUMNS( + id_2 BIGINT, + NESTED PATH 'lax $' AS nested_path_3 COLUMNS(id_3 BIGINT))) + PLAN (root_path OUTER (nested_path_1 CROSS (nested_path_2 UNION nested_path_3)))) + """) + .hasErrorCode(INVALID_PLAN) + .hasMessage("line 10:11: JSON_TABLE plan includes unavailable JSON path names: NESTED_PATH_3"); // nested_path_3 is on another nesting level + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + NESTED PATH 'lax $' AS nested_path_1 COLUMNS(id_1 BIGINT), + NESTED PATH 'lax $' AS nested_path_2 COLUMNS(id_2 BIGINT)) + PLAN (root_path OUTER (nested_path_1 CROSS (nested_path_2 UNION nested_path_1)))) + """) + .hasErrorCode(INVALID_PLAN) + .hasMessage("line 8:69: Duplicate reference to JSON path name in sibling plan: NESTED_PATH_1"); + + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + NESTED PATH 'lax $' AS nested_path_1 COLUMNS(id_1 BIGINT), + NESTED PATH 'lax $' AS nested_path_2 COLUMNS( + id_2 BIGINT, + NESTED PATH 'lax $' AS nested_path_3 COLUMNS(id_3 BIGINT))) + PLAN (root_path OUTER (nested_path_1 CROSS (nested_path_2 INNER nested_path_3)))) + """); + } + + @Test + public void testJsonTableDefaultPlan() + { + analyze(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS(id BIGINT) + PLAN DEFAULT(CROSS, INNER)) + """); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + NESTED PATH 'lax $' COLUMNS(id BIGINT)) + PLAN DEFAULT(OUTER, UNION)) + """) + .hasErrorCode(MISSING_PATH_NAME) + .hasMessage("line 6:21: All nested JSON paths must be named when default plan is given"); + } + + @Test + public void tstJsonTableInJoin() + { + analyze(""" + SELECT * + FROM t1, t2, JSON_TABLE('[1, 2, 3]', 'lax $[2]' COLUMNS(o FOR ORDINALITY)) + """); + + // join condition + analyze(""" + SELECT * + FROM t1 + LEFT JOIN + JSON_TABLE('[1, 2, 3]', 'lax $[2]' COLUMNS(o FOR ORDINALITY)) + ON TRUE + """); + + assertFails(""" + SELECT * + FROM t1 + RIGHT JOIN + JSON_TABLE('[1, 2, 3]', 'lax $[2]' COLUMNS(o FOR ORDINALITY)) t + ON t.o > t1.a + """) .hasErrorCode(NOT_SUPPORTED) - .hasMessage("line 1:15: JSON_TABLE is not yet supported"); + .hasMessage("line 5:12: RIGHT JOIN involving JSON_TABLE is only supported with condition ON TRUE"); + + // correlation in context item + analyze(""" + SELECT * + FROM t6 + LEFT JOIN + JSON_TABLE(b, 'lax $[2]' COLUMNS(o FOR ORDINALITY)) + ON TRUE + """); + + // correlation in default value + analyze(""" + SELECT * + FROM t6 + LEFT JOIN + JSON_TABLE('[1, 2, 3]', 'lax $[2]' COLUMNS(x BIGINT DEFAULT a ON EMPTY)) + ON TRUE + """); + + // correlation in path parameter + analyze(""" + SELECT * + FROM t6 + LEFT JOIN + JSON_TABLE('[1, 2, 3]', 'lax $[2]' PASSING a AS parameter_name COLUMNS(o FOR ORDINALITY)) + ON TRUE + """); + + // invalid correlation in right join + assertFails(""" + SELECT * + FROM t6 + RIGHT JOIN + JSON_TABLE('[1, 2, 3]', 'lax $[2]' PASSING a AS parameter_name COLUMNS(o FOR ORDINALITY)) + ON TRUE + """) + .hasErrorCode(INVALID_COLUMN_REFERENCE) + .hasMessage("line 4:48: LATERAL reference not allowed in RIGHT JOIN"); + } + + @Test + public void testSubqueryInJsonTable() + { + analyze(""" + SELECT * + FROM JSON_TABLE( + (SELECT '[1, 2, 3]'), + 'lax $[2]' PASSING (SELECT 1) AS parameter_name + COLUMNS( + x BIGINT DEFAULT (SELECT 2) ON EMPTY)) + """); + } + + @Test + public void testAggregationInJsonTable() + { + assertFails(""" + SELECT * + FROM JSON_TABLE( + CAST(sum(1) AS varchar), + 'lax $' PASSING 2 AS parameter_name + COLUMNS( + x BIGINT DEFAULT 3 ON EMPTY DEFAULT 4 ON ERROR)) + """) + .hasErrorCode(EXPRESSION_NOT_SCALAR) + .hasMessage("line 3:5: JSON_TABLE input expression cannot contain aggregations, window functions or grouping operations: [sum(1)]"); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '1', + 'lax $' PASSING avg(2) AS parameter_name + COLUMNS( + x BIGINT DEFAULT 3 ON EMPTY DEFAULT 4 ON ERROR)) + """) + .hasErrorCode(EXPRESSION_NOT_SCALAR) + .hasMessage("line 4:21: JSON_TABLE path parameter cannot contain aggregations, window functions or grouping operations: [avg(2)]"); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '1', + 'lax $' PASSING 2 AS parameter_name + COLUMNS( + x BIGINT DEFAULT min(3) ON EMPTY DEFAULT 4 ON ERROR)) + """) + .hasErrorCode(EXPRESSION_NOT_SCALAR) + .hasMessage("line 6:26: default expression for JSON_TABLE column cannot contain aggregations, window functions or grouping operations: [min(3)]"); + + assertFails(""" + SELECT * + FROM JSON_TABLE( + '1', + 'lax $' PASSING 2 AS parameter_name + COLUMNS( + x BIGINT DEFAULT 3 ON EMPTY DEFAULT max(4) ON ERROR)) + """) + .hasErrorCode(EXPRESSION_NOT_SCALAR) + .hasMessage("line 6:45: default expression for JSON_TABLE column cannot contain aggregations, window functions or grouping operations: [max(4)]"); + } + + @Test + public void testAliasJsonTable() + { + analyze(""" + SELECT t.y + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS(x BIGINT)) t(y) + """); + + analyze(""" + SELECT t.x + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $[2]' + COLUMNS(x BIGINT)) t + """); } @Test diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/JsonTablePlanComparator.java b/core/trino-main/src/test/java/io/trino/sql/planner/JsonTablePlanComparator.java new file mode 100644 index 000000000000..8e34adfd2f2d --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/sql/planner/JsonTablePlanComparator.java @@ -0,0 +1,125 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.sql.planner; + +import io.trino.operator.table.json.JsonTableColumn; +import io.trino.operator.table.json.JsonTableOrdinalityColumn; +import io.trino.operator.table.json.JsonTablePlanCross; +import io.trino.operator.table.json.JsonTablePlanLeaf; +import io.trino.operator.table.json.JsonTablePlanNode; +import io.trino.operator.table.json.JsonTablePlanSingle; +import io.trino.operator.table.json.JsonTablePlanUnion; +import io.trino.operator.table.json.JsonTableQueryColumn; +import io.trino.operator.table.json.JsonTableValueColumn; + +import java.util.Comparator; +import java.util.List; + +import static java.util.Objects.requireNonNull; + +public class JsonTablePlanComparator +{ + private JsonTablePlanComparator() {} + + public static Comparator planComparator() + { + return (actual, expected) -> { + requireNonNull(actual, "actual is null"); + requireNonNull(expected, "expected is null"); + return compare(actual, expected) ? 0 : -1; + }; + } + + private static boolean compare(JsonTablePlanNode left, JsonTablePlanNode right) + { + if (left == right) { + return true; + } + if (left.getClass() != right.getClass()) { + return false; + } + if (left instanceof JsonTablePlanLeaf leftPlan) { + JsonTablePlanLeaf rightPlan = (JsonTablePlanLeaf) right; + return leftPlan.path().equals(rightPlan.path()) && + compareColumns(leftPlan.columns(), rightPlan.columns()); + } + if (left instanceof JsonTablePlanSingle leftPlan) { + JsonTablePlanSingle rightPlan = (JsonTablePlanSingle) right; + return leftPlan.path().equals(rightPlan.path()) && + compareColumns(leftPlan.columns(), rightPlan.columns()) && + leftPlan.outer() == rightPlan.outer() && + compare(leftPlan.child(), rightPlan.child()); + } + List leftSiblings; + List rightSiblings; + if (left instanceof JsonTablePlanCross leftPlan) { + leftSiblings = leftPlan.siblings(); + rightSiblings = ((JsonTablePlanCross) right).siblings(); + } + else { + leftSiblings = ((JsonTablePlanUnion) left).siblings(); + rightSiblings = ((JsonTablePlanUnion) right).siblings(); + } + if (leftSiblings.size() != rightSiblings.size()) { + return false; + } + for (int i = 0; i < leftSiblings.size(); i++) { + if (!compare(leftSiblings.get(i), rightSiblings.get(i))) { + return false; + } + } + return true; + } + + private static boolean compareColumns(List leftColumns, List rightColumns) + { + if (leftColumns.size() != rightColumns.size()) { + return false; + } + for (int i = 0; i < leftColumns.size(); i++) { + if (!compareColumn(leftColumns.get(i), rightColumns.get(i))) { + return false; + } + } + return true; + } + + private static boolean compareColumn(JsonTableColumn left, JsonTableColumn right) + { + if (left.getClass() != right.getClass()) { + return false; + } + if (left instanceof JsonTableOrdinalityColumn leftColumn) { + return leftColumn.outputIndex() == ((JsonTableOrdinalityColumn) right).outputIndex(); + } + if (left instanceof JsonTableQueryColumn leftColumn) { + JsonTableQueryColumn rightColumn = (JsonTableQueryColumn) right; + return leftColumn.outputIndex() == rightColumn.outputIndex() && + leftColumn.function().equals(rightColumn.function()) && + leftColumn.path().equals(rightColumn.path()) && + leftColumn.wrapperBehavior() == rightColumn.wrapperBehavior() && + leftColumn.emptyBehavior() == rightColumn.emptyBehavior() && + leftColumn.errorBehavior() == rightColumn.errorBehavior(); + } + JsonTableValueColumn leftColumn = (JsonTableValueColumn) left; + JsonTableValueColumn rightColumn = (JsonTableValueColumn) right; + return leftColumn.outputIndex() == rightColumn.outputIndex() && + leftColumn.function().equals(rightColumn.function()) && + leftColumn.path().equals(rightColumn.path()) && + leftColumn.emptyBehavior() == rightColumn.emptyBehavior() && + leftColumn.emptyDefaultInput() == rightColumn.emptyDefaultInput() && + leftColumn.errorBehavior() == rightColumn.errorBehavior() && + leftColumn.errorDefaultInput() == rightColumn.errorDefaultInput(); + } +} diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/TestJsonTable.java b/core/trino-main/src/test/java/io/trino/sql/planner/TestJsonTable.java new file mode 100644 index 000000000000..4631acea7921 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/sql/planner/TestJsonTable.java @@ -0,0 +1,549 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.sql.planner; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import io.trino.execution.warnings.WarningCollector; +import io.trino.json.ir.IrJsonPath; +import io.trino.metadata.ResolvedFunction; +import io.trino.metadata.TestingFunctionResolution; +import io.trino.operator.table.json.JsonTable; +import io.trino.operator.table.json.JsonTablePlanCross; +import io.trino.operator.table.json.JsonTablePlanLeaf; +import io.trino.operator.table.json.JsonTablePlanNode; +import io.trino.operator.table.json.JsonTablePlanSingle; +import io.trino.operator.table.json.JsonTablePlanUnion; +import io.trino.operator.table.json.JsonTableQueryColumn; +import io.trino.operator.table.json.JsonTableValueColumn; +import io.trino.sql.planner.assertions.BasePlanTest; +import io.trino.sql.planner.optimizations.PlanNodeSearcher; +import io.trino.sql.planner.plan.TableFunctionNode; +import io.trino.sql.tree.JsonQuery; +import io.trino.sql.tree.JsonValue; +import org.intellij.lang.annotations.Language; +import org.junit.jupiter.api.Test; + +import static com.google.common.collect.Iterables.getOnlyElement; +import static io.trino.execution.querystats.PlanOptimizersStatsCollector.createPlanOptimizersStatsCollector; +import static io.trino.operator.scalar.json.JsonQueryFunction.JSON_QUERY_FUNCTION_NAME; +import static io.trino.operator.scalar.json.JsonValueFunction.JSON_VALUE_FUNCTION_NAME; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.BooleanType.BOOLEAN; +import static io.trino.spi.type.TinyintType.TINYINT; +import static io.trino.sql.analyzer.ExpressionAnalyzer.JSON_NO_PARAMETERS_ROW_TYPE; +import static io.trino.sql.analyzer.TypeSignatureProvider.fromTypes; +import static io.trino.sql.planner.JsonTablePlanComparator.planComparator; +import static io.trino.sql.planner.LogicalPlanner.Stage.CREATED; +import static io.trino.sql.planner.PathNodes.contextVariable; +import static io.trino.sql.planner.PathNodes.literal; +import static io.trino.sql.planner.PathNodes.memberAccessor; +import static io.trino.sql.planner.assertions.PlanMatchPattern.anyTree; +import static io.trino.sql.planner.assertions.PlanMatchPattern.expression; +import static io.trino.sql.planner.assertions.PlanMatchPattern.project; +import static io.trino.sql.planner.assertions.PlanMatchPattern.strictOutput; +import static io.trino.sql.planner.assertions.PlanMatchPattern.tableFunction; +import static io.trino.sql.planner.assertions.PlanMatchPattern.values; +import static io.trino.sql.planner.assertions.TableFunctionMatcher.TableArgumentValue.Builder.tableArgument; +import static io.trino.type.Json2016Type.JSON_2016; +import static io.trino.type.TestJsonPath2016TypeSerialization.JSON_PATH_2016; +import static org.assertj.core.api.Assertions.assertThat; + +public class TestJsonTable + extends BasePlanTest +{ + private static final ResolvedFunction JSON_VALUE_FUNCTION = new TestingFunctionResolution().resolveFunction( + JSON_VALUE_FUNCTION_NAME, + fromTypes(JSON_2016, JSON_PATH_2016, JSON_NO_PARAMETERS_ROW_TYPE, TINYINT, BIGINT, TINYINT, BIGINT)); + + private static final ResolvedFunction JSON_QUERY_FUNCTION = new TestingFunctionResolution().resolveFunction( + JSON_QUERY_FUNCTION_NAME, + fromTypes(JSON_2016, JSON_PATH_2016, JSON_NO_PARAMETERS_ROW_TYPE, TINYINT, TINYINT, TINYINT)); + + @Test + public void testJsonTableInitialPlan() + { + assertPlan( + """ + SELECT * + FROM (SELECT '[1, 2, 3]', 4) t(json_col, int_col), JSON_TABLE( + json_col, + 'lax $' AS root_path PASSING int_col AS id, '[ala]' FORMAT JSON AS name + COLUMNS( + bigint_col BIGINT DEFAULT 5 ON EMPTY DEFAULT int_col ON ERROR, + varchar_col VARCHAR FORMAT JSON ERROR ON ERROR) + EMPTY ON ERROR) + """, + CREATED, + strictOutput(// left-side columns first, json_table columns next + ImmutableList.of("json_col", "int_col", "bigint_col", "formatted_varchar_col"), + anyTree( + project( + ImmutableMap.of("formatted_varchar_col", expression("\"$json_to_varchar\"(varchar_col, tinyint '1', false)")), + tableFunction(builder -> builder + .name("$json_table") + .addTableArgument( + "$input", + tableArgument(0) + .rowSemantics() + .passThroughColumns() + .passThroughSymbols(ImmutableSet.of("json_col", "int_col"))) + .properOutputs(ImmutableList.of("bigint_col", "varchar_col")), + project( + ImmutableMap.of( + "context_item", expression("\"$varchar_to_json\"(json_col_coerced, false)"), // apply input function to context item + "parameters_row", expression("CAST(ROW (int_col, \"$varchar_to_json\"(name_coerced, false)) AS ROW(ID integer, NAME json2016))")), // apply input function to formatted path parameter and gather path parameters in a row + project(// coerce context item, path parameters and default expressions + ImmutableMap.of( + "name_coerced", expression("CAST(name AS VARCHAR)"), // cast formatted path parameter to VARCHAR for the input function + "default_value_coerced", expression("CAST(default_value AS BIGINT)"), // cast default value to BIGINT to match declared return type for the column + "json_col_coerced", expression("CAST(json_col AS VARCHAR)"), // cast context item to VARCHAR for the input function + "int_col_coerced", expression("CAST(int_col AS BIGINT)")), // cast default value to BIGINT to match declared return type for the column + project(// pre-project context item, path parameters and default expressions + ImmutableMap.of( + "name", expression("'[ala]'"), + "default_value", expression("5")), + anyTree( + project( + ImmutableMap.of( + "json_col", expression("'[1, 2, 3]'"), + "int_col", expression("4")), + values(1))))))))))); + } + + @Test + public void testImplicitColumnPath() + { + assertJsonTablePlan( + """ + SELECT * + FROM (SELECT 1, 2, 3), JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + first_col BIGINT, + "Second_Col" BIGINT, + "_""_'_?_" BIGINT)) + """, + new JsonTablePlanLeaf( + new IrJsonPath(true, contextVariable()), + ImmutableList.of( + valueColumn(0, new IrJsonPath(true, memberAccessor(contextVariable(), "FIRST_COL"))), + valueColumn(1, new IrJsonPath(true, memberAccessor(contextVariable(), "Second_Col"))), + valueColumn(2, new IrJsonPath(true, memberAccessor(contextVariable(), "_\"_'_?_")))))); + } + + @Test + public void testExplicitColumnPath() + { + assertJsonTablePlan( + """ + SELECT * + FROM (SELECT 1, 2, 3), JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + first_col BIGINT PATH 'lax $.a', + "Second_Col" BIGINT PATH 'lax $.B', + "_""_'_?_" BIGINT PATH 'lax false')) + """, + new JsonTablePlanLeaf( + new IrJsonPath(true, contextVariable()), + ImmutableList.of( + valueColumn(0, new IrJsonPath(true, memberAccessor(contextVariable(), "a"))), + valueColumn(1, new IrJsonPath(true, memberAccessor(contextVariable(), "B"))), + valueColumn(2, new IrJsonPath(true, literal(BOOLEAN, false)))))); + } + + @Test + public void testColumnOutputIndex() + { + // output indexes follow the declaration order: [a, b, c, d] + assertJsonTablePlan( + """ + SELECT * + FROM (SELECT 1, 2, 3), JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + a BIGINT, + NESTED PATH 'lax $.x' COLUMNS( + b BIGINT, + NESTED PATH 'lax $.y' COLUMNS( + c BIGINT)), + d BIGINT)) + """, + new JsonTablePlanSingle( + new IrJsonPath(true, contextVariable()), + ImmutableList.of( + valueColumn(0, new IrJsonPath(true, memberAccessor(contextVariable(), "A"))), + valueColumn(3, new IrJsonPath(true, memberAccessor(contextVariable(), "D")))), + true, + new JsonTablePlanSingle( + new IrJsonPath(true, memberAccessor(contextVariable(), "x")), + ImmutableList.of(valueColumn(1, new IrJsonPath(true, memberAccessor(contextVariable(), "B")))), + true, + new JsonTablePlanLeaf( + new IrJsonPath(true, memberAccessor(contextVariable(), "y")), + ImmutableList.of(valueColumn(2, new IrJsonPath(true, memberAccessor(contextVariable(), "C")))))))); + } + + @Test + public void testColumnBehavior() + { + assertJsonTablePlan( + """ + SELECT * + FROM (SELECT 1, 2, 3), JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + a BIGINT, + b BIGINT NULL ON EMPTY ERROR ON ERROR, + c BIGINT DEFAULT 1 ON EMPTY DEFAULT 2 ON ERROR, + d VARCHAR FORMAT JSON, + e VARCHAR FORMAT JSON WITH CONDITIONAL ARRAY WRAPPER NULL ON EMPTY ERROR ON ERROR, + f VARCHAR FORMAT JSON OMIT QUOTES EMPTY ARRAY ON EMPTY EMPTY OBJECT ON ERROR)) + """, + new JsonTablePlanLeaf( + new IrJsonPath(true, contextVariable()), + ImmutableList.of( + valueColumn( + 0, + new IrJsonPath(true, memberAccessor(contextVariable(), "A")), + JsonValue.EmptyOrErrorBehavior.NULL, + -1, + JsonValue.EmptyOrErrorBehavior.NULL, + -1), + valueColumn( + 1, + new IrJsonPath(true, memberAccessor(contextVariable(), "B")), + JsonValue.EmptyOrErrorBehavior.NULL, + -1, + JsonValue.EmptyOrErrorBehavior.ERROR, + -1), + valueColumn( + 2, + new IrJsonPath(true, memberAccessor(contextVariable(), "C")), + JsonValue.EmptyOrErrorBehavior.DEFAULT, + 2, + JsonValue.EmptyOrErrorBehavior.DEFAULT, + 3), + queryColumn( + 3, + new IrJsonPath(true, memberAccessor(contextVariable(), "D")), + JsonQuery.ArrayWrapperBehavior.WITHOUT, + JsonQuery.EmptyOrErrorBehavior.NULL, + JsonQuery.EmptyOrErrorBehavior.NULL), + queryColumn( + 4, + new IrJsonPath(true, memberAccessor(contextVariable(), "E")), + JsonQuery.ArrayWrapperBehavior.CONDITIONAL, + JsonQuery.EmptyOrErrorBehavior.NULL, + JsonQuery.EmptyOrErrorBehavior.ERROR), + queryColumn( + 5, + new IrJsonPath(true, memberAccessor(contextVariable(), "F")), + JsonQuery.ArrayWrapperBehavior.WITHOUT, + JsonQuery.EmptyOrErrorBehavior.EMPTY_ARRAY, + JsonQuery.EmptyOrErrorBehavior.EMPTY_OBJECT)))); + } + + @Test + public void testInheritedErrorBehavior() + { + // the column has no explicit error behavior, and json_table has no explicit error behavior. The default behavior for column is NULL ON ERROR. + assertJsonTablePlan( + """ + SELECT * + FROM (SELECT 1, 2, 3), JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + a BIGINT)) + """, + new JsonTablePlanLeaf( + new IrJsonPath(true, contextVariable()), + ImmutableList.of( + valueColumn( + 0, + new IrJsonPath(true, memberAccessor(contextVariable(), "A")), + JsonValue.EmptyOrErrorBehavior.NULL, + -1, + JsonValue.EmptyOrErrorBehavior.NULL, + -1)))); + + // the column has no explicit error behavior, and json_table has explicit ERROR ON ERROR. The default behavior for column is ERROR ON ERROR. + assertJsonTablePlan( + """ + SELECT * + FROM (SELECT 1, 2, 3), JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + a BIGINT) + ERROR ON ERROR) + """, + new JsonTablePlanLeaf( + new IrJsonPath(true, contextVariable()), + ImmutableList.of( + valueColumn( + 0, + new IrJsonPath(true, memberAccessor(contextVariable(), "A")), + JsonValue.EmptyOrErrorBehavior.NULL, + -1, + JsonValue.EmptyOrErrorBehavior.ERROR, + -1)))); + + // the column has no explicit error behavior, and json_table has explicit EMPTY ON ERROR. The default behavior for column is NULL ON ERROR. + assertJsonTablePlan( + """ + SELECT * + FROM (SELECT 1, 2, 3), JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + a BIGINT) + EMPTY ON ERROR) + """, + new JsonTablePlanLeaf( + new IrJsonPath(true, contextVariable()), + ImmutableList.of( + valueColumn( + 0, + new IrJsonPath(true, memberAccessor(contextVariable(), "A")), + JsonValue.EmptyOrErrorBehavior.NULL, + -1, + JsonValue.EmptyOrErrorBehavior.NULL, + -1)))); + + // the column has explicit NULL ON ERROR behavior, and json_table has no explicit ERROR ON ERROR. The behavior for column is the one explicitly specified. + assertJsonTablePlan( + """ + SELECT * + FROM (SELECT 1, 2, 3), JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + a BIGINT NULL ON ERROR) + ERROR ON ERROR) + """, + new JsonTablePlanLeaf( + new IrJsonPath(true, contextVariable()), + ImmutableList.of( + valueColumn( + 0, + new IrJsonPath(true, memberAccessor(contextVariable(), "A")), + JsonValue.EmptyOrErrorBehavior.NULL, + -1, + JsonValue.EmptyOrErrorBehavior.NULL, + -1)))); + } + + @Test + public void testImplicitDefaultPlan() + { + // implicit plan settings are OUTER, UNION + assertJsonTablePlan( + """ + SELECT * + FROM (SELECT 1, 2, 3), JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + NESTED PATH 'lax $.a' COLUMNS(col_1 BIGINT), + NESTED PATH 'lax $.b' COLUMNS( + NESTED PATH 'lax $.c' COLUMNS(col_2 BIGINT), + NESTED PATH 'lax $.d' COLUMNS(col_3 BIGINT)), + NESTED PATH 'lax $.e' COLUMNS(col_4 BIGINT))) + """, + new JsonTablePlanSingle( + new IrJsonPath(true, contextVariable()), + ImmutableList.of(), + true, + new JsonTablePlanUnion(ImmutableList.of( + new JsonTablePlanLeaf( + new IrJsonPath(true, memberAccessor(contextVariable(), "a")), + ImmutableList.of(valueColumn(0, new IrJsonPath(true, memberAccessor(contextVariable(), "COL_1"))))), + new JsonTablePlanSingle( + new IrJsonPath(true, memberAccessor(contextVariable(), "b")), + ImmutableList.of(), + true, + new JsonTablePlanUnion(ImmutableList.of( + new JsonTablePlanLeaf( + new IrJsonPath(true, memberAccessor(contextVariable(), "c")), + ImmutableList.of(valueColumn(1, new IrJsonPath(true, memberAccessor(contextVariable(), "COL_2"))))), + new JsonTablePlanLeaf( + new IrJsonPath(true, memberAccessor(contextVariable(), "d")), + ImmutableList.of(valueColumn(2, new IrJsonPath(true, memberAccessor(contextVariable(), "COL_3")))))))), + new JsonTablePlanLeaf( + new IrJsonPath(true, memberAccessor(contextVariable(), "e")), + ImmutableList.of(valueColumn(3, new IrJsonPath(true, memberAccessor(contextVariable(), "COL_4"))))))))); + } + + @Test + public void testExplicitDefaultPlan() + { + assertJsonTablePlan( + """ + SELECT * + FROM (SELECT 1, 2, 3), JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + NESTED PATH 'lax $.a' AS a COLUMNS(col_1 BIGINT), + NESTED PATH 'lax $.b' AS b COLUMNS( + NESTED PATH 'lax $.c' AS c COLUMNS(col_2 BIGINT), + NESTED PATH 'lax $.d' AS d COLUMNS(col_3 BIGINT)), + NESTED PATH 'lax $.e' AS e COLUMNS(col_4 BIGINT)) + PLAN DEFAULT (INNER, CROSS)) + """, + new JsonTablePlanSingle( + new IrJsonPath(true, contextVariable()), + ImmutableList.of(), + false, + new JsonTablePlanCross(ImmutableList.of( + new JsonTablePlanLeaf( + new IrJsonPath(true, memberAccessor(contextVariable(), "a")), + ImmutableList.of(valueColumn(0, new IrJsonPath(true, memberAccessor(contextVariable(), "COL_1"))))), + new JsonTablePlanSingle( + new IrJsonPath(true, memberAccessor(contextVariable(), "b")), + ImmutableList.of(), + false, + new JsonTablePlanCross(ImmutableList.of( + new JsonTablePlanLeaf( + new IrJsonPath(true, memberAccessor(contextVariable(), "c")), + ImmutableList.of(valueColumn(1, new IrJsonPath(true, memberAccessor(contextVariable(), "COL_2"))))), + new JsonTablePlanLeaf( + new IrJsonPath(true, memberAccessor(contextVariable(), "d")), + ImmutableList.of(valueColumn(2, new IrJsonPath(true, memberAccessor(contextVariable(), "COL_3")))))))), + new JsonTablePlanLeaf( + new IrJsonPath(true, memberAccessor(contextVariable(), "e")), + ImmutableList.of(valueColumn(3, new IrJsonPath(true, memberAccessor(contextVariable(), "COL_4"))))))))); + + assertJsonTablePlan( + """ + SELECT * + FROM (SELECT 1, 2, 3), JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + NESTED PATH 'lax $.a' AS a COLUMNS(col_1 BIGINT), + NESTED PATH 'lax $.b' AS b COLUMNS( + NESTED PATH 'lax $.c' AS c COLUMNS(col_2 BIGINT), + NESTED PATH 'lax $.d' AS d COLUMNS(col_3 BIGINT)), + NESTED PATH 'lax $.e' AS e COLUMNS(col_4 BIGINT)) + PLAN DEFAULT (CROSS)) + """, + new JsonTablePlanSingle( + new IrJsonPath(true, contextVariable()), + ImmutableList.of(), + true, + new JsonTablePlanCross(ImmutableList.of( + new JsonTablePlanLeaf( + new IrJsonPath(true, memberAccessor(contextVariable(), "a")), + ImmutableList.of(valueColumn(0, new IrJsonPath(true, memberAccessor(contextVariable(), "COL_1"))))), + new JsonTablePlanSingle( + new IrJsonPath(true, memberAccessor(contextVariable(), "b")), + ImmutableList.of(), + true, + new JsonTablePlanCross(ImmutableList.of( + new JsonTablePlanLeaf( + new IrJsonPath(true, memberAccessor(contextVariable(), "c")), + ImmutableList.of(valueColumn(1, new IrJsonPath(true, memberAccessor(contextVariable(), "COL_2"))))), + new JsonTablePlanLeaf( + new IrJsonPath(true, memberAccessor(contextVariable(), "d")), + ImmutableList.of(valueColumn(2, new IrJsonPath(true, memberAccessor(contextVariable(), "COL_3")))))))), + new JsonTablePlanLeaf( + new IrJsonPath(true, memberAccessor(contextVariable(), "e")), + ImmutableList.of(valueColumn(3, new IrJsonPath(true, memberAccessor(contextVariable(), "COL_4"))))))))); + } + + @Test + public void testSpecificPlan() + { + assertJsonTablePlan( + """ + SELECT * + FROM (SELECT 1, 2, 3), JSON_TABLE( + '[1, 2, 3]', + 'lax $' AS root_path + COLUMNS( + NESTED PATH 'lax $.a' AS a COLUMNS(col_1 BIGINT), + NESTED PATH 'lax $.b' AS b COLUMNS( + NESTED PATH 'lax $.c' AS c COLUMNS(col_2 BIGINT), + NESTED PATH 'lax $.d' AS d COLUMNS(col_3 BIGINT)), + NESTED PATH 'lax $.e' AS e COLUMNS(col_4 BIGINT)) + PLAN (ROOT_PATH INNER (((B OUTER (D CROSS C)) UNION E) CROSS A))) + """, + new JsonTablePlanSingle( + new IrJsonPath(true, contextVariable()), + ImmutableList.of(), + false, + new JsonTablePlanCross(ImmutableList.of( + new JsonTablePlanUnion(ImmutableList.of( + new JsonTablePlanSingle( + new IrJsonPath(true, memberAccessor(contextVariable(), "b")), + ImmutableList.of(), + true, + new JsonTablePlanCross(ImmutableList.of( + new JsonTablePlanLeaf( + new IrJsonPath(true, memberAccessor(contextVariable(), "d")), + ImmutableList.of(valueColumn(2, new IrJsonPath(true, memberAccessor(contextVariable(), "COL_3"))))), + new JsonTablePlanLeaf( + new IrJsonPath(true, memberAccessor(contextVariable(), "c")), + ImmutableList.of(valueColumn(1, new IrJsonPath(true, memberAccessor(contextVariable(), "COL_2")))))))), + new JsonTablePlanLeaf( + new IrJsonPath(true, memberAccessor(contextVariable(), "e")), + ImmutableList.of(valueColumn(3, new IrJsonPath(true, memberAccessor(contextVariable(), "COL_4"))))))), + new JsonTablePlanLeaf( + new IrJsonPath(true, memberAccessor(contextVariable(), "a")), + ImmutableList.of(valueColumn(0, new IrJsonPath(true, memberAccessor(contextVariable(), "COL_1"))))))))); + } + + private static JsonTableValueColumn valueColumn(int outputIndex, IrJsonPath path) + { + return valueColumn(outputIndex, path, JsonValue.EmptyOrErrorBehavior.NULL, -1, JsonValue.EmptyOrErrorBehavior.NULL, -1); + } + + private static JsonTableValueColumn valueColumn(int outputIndex, IrJsonPath path, JsonValue.EmptyOrErrorBehavior emptyBehavior, int emptyDefaultInput, JsonValue.EmptyOrErrorBehavior errorBehavior, int errorDefaultInput) + { + return new JsonTableValueColumn(outputIndex, JSON_VALUE_FUNCTION, path, emptyBehavior.ordinal(), emptyDefaultInput, errorBehavior.ordinal(), errorDefaultInput); + } + + private static JsonTableQueryColumn queryColumn(int outputIndex, IrJsonPath path, JsonQuery.ArrayWrapperBehavior wrapperBehavior, JsonQuery.EmptyOrErrorBehavior emptyBehavior, JsonQuery.EmptyOrErrorBehavior errorBehavior) + { + return new JsonTableQueryColumn(outputIndex, JSON_QUERY_FUNCTION, path, wrapperBehavior.ordinal(), emptyBehavior.ordinal(), errorBehavior.ordinal()); + } + + private void assertJsonTablePlan(@Language("SQL") String sql, JsonTablePlanNode expectedPlan) + { + try { + getQueryRunner().inTransaction(transactionSession -> { + Plan queryPlan = getQueryRunner().createPlan(transactionSession, sql, ImmutableList.of(), CREATED, WarningCollector.NOOP, createPlanOptimizersStatsCollector()); + TableFunctionNode tableFunctionNode = getOnlyElement(PlanNodeSearcher.searchFrom(queryPlan.getRoot()).where(TableFunctionNode.class::isInstance).findAll()); + JsonTablePlanNode actualPlan = ((JsonTable.JsonTableFunctionHandle) tableFunctionNode.getHandle().getFunctionHandle()).processingPlan(); + assertThat(actualPlan) + .usingComparator(planComparator()) + .isEqualTo(expectedPlan); + return null; + }); + } + catch (Throwable e) { + e.addSuppressed(new Exception("Query: " + sql)); + throw e; + } + } +} diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/TestingPlannerContext.java b/core/trino-main/src/test/java/io/trino/sql/planner/TestingPlannerContext.java index a7e8f9bc4bde..f2e8d9647efd 100644 --- a/core/trino-main/src/test/java/io/trino/sql/planner/TestingPlannerContext.java +++ b/core/trino-main/src/test/java/io/trino/sql/planner/TestingPlannerContext.java @@ -119,7 +119,10 @@ public PlannerContext build() types.forEach(typeRegistry::addType); parametricTypes.forEach(typeRegistry::addParametricType); - GlobalFunctionCatalog globalFunctionCatalog = new GlobalFunctionCatalog(); + GlobalFunctionCatalog globalFunctionCatalog = new GlobalFunctionCatalog( + () -> { throw new UnsupportedOperationException(); }, + () -> { throw new UnsupportedOperationException(); }, + () -> { throw new UnsupportedOperationException(); }); globalFunctionCatalog.addFunctions(SystemFunctionBundle.create(featuresConfig, typeOperators, new BlockTypeOperators(typeOperators), UNKNOWN)); functionBundles.forEach(globalFunctionCatalog::addFunctions); diff --git a/core/trino-main/src/test/java/io/trino/sql/query/TestJsonTable.java b/core/trino-main/src/test/java/io/trino/sql/query/TestJsonTable.java new file mode 100644 index 000000000000..c5f15ed66205 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/sql/query/TestJsonTable.java @@ -0,0 +1,867 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.sql.query; + +import io.trino.Session; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import static com.google.common.io.BaseEncoding.base16; +import static io.trino.spi.StandardErrorCode.PATH_EVALUATION_ERROR; +import static io.trino.testing.assertions.TrinoExceptionAssert.assertTrinoExceptionThrownBy; +import static java.nio.charset.StandardCharsets.UTF_16LE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; + +@TestInstance(PER_CLASS) +public class TestJsonTable +{ + private QueryAssertions assertions; + + @BeforeAll + public void init() + { + assertions = new QueryAssertions(); + } + + @AfterAll + public void teardown() + { + assertions.close(); + assertions = null; + } + + @Test + public void testSimple() + { + assertThat(assertions.query(""" + SELECT first, last + FROM (SELECT '{"a" : [1, 2, 3], "b" : [4, 5, 6]}') t(json_col), JSON_TABLE( + json_col, + 'lax $.a' + COLUMNS( + first bigint PATH 'lax $[0]', + last bigint PATH 'lax $[last]')) + """)) + .matches("VALUES (BIGINT '1', BIGINT '3')"); + + assertThat(assertions.query(""" + SELECT * + FROM + (SELECT '{"a" : {"b" : [1, 2, 3], "c" : [[4, 5, 6], [7, 8, 9]]}}') t(json_col), + JSON_TABLE( + json_col, + 'lax $.a' AS "path_a" + COLUMNS( + NESTED PATH 'lax $.b[*]' AS "path_b" + COLUMNS (c1 integer PATH 'lax $ * 10'), + NESTED PATH 'lax $.c' AS "path_c" + COLUMNS ( + NESTED PATH 'lax $[0][*]' AS "path_d" COLUMNS (c2 integer PATH 'lax $ * 100'), + NESTED PATH 'lax $[last][*]' AS "path_e" COLUMNS (c3 integer PATH 'lax $ * 1000'))) + PLAN ("path_a" OUTER ("path_b" UNION ("path_c" INNER ("path_d" CROSS "path_e"))))) + """)) + .matches(""" + VALUES + ('{"a" : {"b" : [1, 2, 3], "c" : [[4, 5, 6], [7, 8, 9]]}}', 10, CAST(null AS integer), CAST(null AS integer)), + ('{"a" : {"b" : [1, 2, 3], "c" : [[4, 5, 6], [7, 8, 9]]}}', 20, null, null), + ('{"a" : {"b" : [1, 2, 3], "c" : [[4, 5, 6], [7, 8, 9]]}}', 30, null, null), + ('{"a" : {"b" : [1, 2, 3], "c" : [[4, 5, 6], [7, 8, 9]]}}', null, 400, 7000), + ('{"a" : {"b" : [1, 2, 3], "c" : [[4, 5, 6], [7, 8, 9]]}}', null, 400, 8000), + ('{"a" : {"b" : [1, 2, 3], "c" : [[4, 5, 6], [7, 8, 9]]}}', null, 400, 9000), + ('{"a" : {"b" : [1, 2, 3], "c" : [[4, 5, 6], [7, 8, 9]]}}', null, 500, 7000), + ('{"a" : {"b" : [1, 2, 3], "c" : [[4, 5, 6], [7, 8, 9]]}}', null, 500, 8000), + ('{"a" : {"b" : [1, 2, 3], "c" : [[4, 5, 6], [7, 8, 9]]}}', null, 500, 9000), + ('{"a" : {"b" : [1, 2, 3], "c" : [[4, 5, 6], [7, 8, 9]]}}', null, 600, 7000), + ('{"a" : {"b" : [1, 2, 3], "c" : [[4, 5, 6], [7, 8, 9]]}}', null, 600, 8000), + ('{"a" : {"b" : [1, 2, 3], "c" : [[4, 5, 6], [7, 8, 9]]}}', null, 600, 9000) + """); + } + + @Test + public void testSubqueries() + { + // test subqueries in: context item, value of path parameter "index", empty default, error default + assertThat(assertions.query(""" + SELECT empty_default, error_default + FROM (SELECT '[[1, 2, 3], [4, 5, 6]]') t(json_col), JSON_TABLE( + (SELECT json_col), + 'lax $[$index]' PASSING (SELECT 0) AS "index" + COLUMNS( + empty_default bigint PATH 'lax $[-42]' DEFAULT (SELECT -42) ON EMPTY, + error_default bigint PATH 'strict $[42]' DEFAULT (SELECT 42) ON ERROR)) + """)) + .matches("VALUES (BIGINT '-42', BIGINT '42')"); + } + + @Test + public void testCorrelation() + { + // test correlation in: context item, value of path parameter "index", empty default, error default + assertThat(assertions.query(""" + SELECT empty_default, error_default + FROM (SELECT '[[1, 2, 3], [4, 5, 6]]', 0, -42, 42) t(json_col, index_col, empty_default_col, error_default_col), + JSON_TABLE( + json_col, + 'lax $[$index]' PASSING index_col AS "index" + COLUMNS( + empty_default bigint PATH 'lax $[-42]' DEFAULT empty_default_col ON EMPTY, + error_default bigint PATH 'strict $[42]' DEFAULT error_default_col ON ERROR)) + """)) + .matches("VALUES (BIGINT '-42', BIGINT '42')"); + } + + @Test + public void testParameters() + { + // test parameters in: context item, value of path parameter "index", empty default, error default + Session session = Session.builder(assertions.getDefaultSession()) + .addPreparedStatement( + "my_query", + """ + SELECT empty_default, error_default + FROM JSON_TABLE( + ?, + 'lax $[$index]' PASSING ? AS "index" + COLUMNS( + empty_default bigint PATH 'lax $[-42]' DEFAULT ? ON EMPTY, + error_default bigint PATH 'strict $[42]' DEFAULT ? ON ERROR)) + """) + .build(); + assertThat(assertions.query(session, "EXECUTE my_query USING '[[1, 2, 3], [4, 5, 6]]', 0, -42, 42")) + .matches("VALUES (BIGINT '-42', BIGINT '42')"); + } + + @Test + public void testOutputLayout() + { + // first the columns from the left side of the join (json_col, index_col, empty_default_col, error_default_col), next the json_table columns (empty_default, error_default) + assertThat(assertions.query(""" + SELECT * + FROM (SELECT '[[1, 2, 3], [4, 5, 6]]', 0, -42, 42) t(json_col, index_col, empty_default_col, error_default_col), + JSON_TABLE( + json_col, + 'lax $[$index]' PASSING index_col AS "index" + COLUMNS( + empty_default bigint PATH 'lax $[-42]' DEFAULT empty_default_col * 2 ON EMPTY, + error_default bigint PATH 'strict $[42]' DEFAULT error_default_col * 2 ON ERROR)) + """)) + .matches("VALUES ('[[1, 2, 3], [4, 5, 6]]', 0, -42, 42, BIGINT '-84', BIGINT '84')"); + + // json_table columns in order of declaration + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[]', + 'lax $' AS "p" + COLUMNS( + a varchar(1) PATH 'lax "A"', + NESTED PATH 'lax $' AS "p1" + COLUMNS ( + b varchar(1) PATH 'lax "B"', + NESTED PATH 'lax $' AS "p2 "COLUMNS ( + c varchar(1) PATH 'lax "C"', + d varchar(1) PATH 'lax "D"'), + e varchar(1) PATH 'lax "E"'), + f varchar(1) PATH 'lax "F"', + NESTED PATH 'lax $' AS "p3" + COLUMNS (g varchar(1) PATH 'lax "G"'), + h varchar(1) PATH 'lax "H"') + PLAN DEFAULT (CROSS)) + """)) + .matches("VALUES ('A', 'B', 'C', 'D', 'E', 'F', 'G', 'H')"); + } + + @Test + public void testJoinTypes() + { + // implicit CROSS join + assertThat(assertions.query(""" + SELECT * + FROM (VALUES ('[1, 2, 3]'), ('[4, 5, 6, 7, 8]')) t(json_col), + JSON_TABLE( + json_col, + 'lax $[4]' + COLUMNS(a integer PATH 'lax $')) + """)) + .matches("VALUES ('[4, 5, 6, 7, 8]', 8)"); + + // INNER join + assertThat(assertions.query(""" + SELECT * + FROM (VALUES ('[1, 2, 3]'), ('[4, 5, 6, 7, 8]')) t(json_col) + INNER JOIN + JSON_TABLE( + json_col, + 'lax $[4]' + COLUMNS(a integer PATH 'lax $')) + ON TRUE + """)) + .matches("VALUES ('[4, 5, 6, 7, 8]', 8)"); + + // LEFT join + assertThat(assertions.query(""" + SELECT * + FROM (VALUES ('[1, 2, 3]'), ('[4, 5, 6, 7, 8]')) t(json_col) + LEFT JOIN + JSON_TABLE( + json_col, + 'lax $[4]' + COLUMNS(a integer PATH 'lax $')) + ON TRUE + """)) + .matches(""" + VALUES + ('[1, 2, 3]', CAST(null AS integer)), + ('[4, 5, 6, 7, 8]', 8) + """); + + // RIGHT join is effectively INNER. Correlation is not allowed in RIGHT join + assertThat(assertions.query(""" + SELECT * + FROM (VALUES 1) t(x) + RIGHT JOIN + JSON_TABLE( + '[1, 2, 3]', + 'lax $[4]' + COLUMNS(a integer PATH 'lax $')) + ON TRUE + """)) + .returnsEmptyResult(); + + // FULL join. Correlation is not allowed in FULL join + assertThat(assertions.query(""" + SELECT * + FROM (VALUES 1) t(x) + FULL JOIN + JSON_TABLE( + '[1, 2, 3]', + 'lax $[4]' + COLUMNS(a integer PATH 'lax $')) + ON TRUE + """)) + .matches("VALUES (1, CAST(null AS integer))"); + } + + @Test + public void testParentChildRelationship() + { + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[]', + 'lax $' AS "root_path" + COLUMNS( + a varchar(1) PATH 'lax "A"', + NESTED PATH 'lax $[*]' AS "nested_path" + COLUMNS (b varchar(1) PATH 'lax "B"')) + PLAN ("root_path" OUTER "nested_path")) + """)) + .matches("VALUES ('A', CAST(null AS varchar(1)))"); + + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[]', + 'lax $' AS "root_path" + COLUMNS( + a varchar(1) PATH 'lax "A"', + NESTED PATH 'lax $[*]' AS "nested_path" + COLUMNS (b varchar(1) PATH 'lax "B"')) + PLAN ("root_path" INNER "nested_path")) + """)) + .returnsEmptyResult(); + + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[[], [1]]', + 'lax $' AS "root_path" + COLUMNS( + a varchar(1) PATH 'lax "A"', + NESTED PATH 'lax $[*]' AS "nested_path_1" + COLUMNS ( + b varchar(1) PATH 'lax "B"', + NESTED PATH 'lax $[*]' AS "nested_path_2" + COLUMNS( + c varchar(1) PATH 'lax "C"'))) + PLAN ("root_path" OUTER ("nested_path_1" OUTER "nested_path_2"))) + """)) + .matches(""" + VALUES + ('A', 'B', CAST(null AS varchar(1))), + ('A', 'B', 'C') + """); + + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[[], [1]]', + 'lax $' AS "root_path" + COLUMNS( + a varchar(1) PATH 'lax "A"', + NESTED PATH 'lax $[*]' AS "nested_path_1" + COLUMNS ( + b varchar(1) PATH 'lax "B"', + NESTED PATH 'lax $[*]' AS "nested_path_2" + COLUMNS( + c varchar(1) PATH 'lax "C"'))) + PLAN ("root_path" OUTER ("nested_path_1" INNER "nested_path_2"))) + """)) + .matches("VALUES ('A', 'B', 'C')"); + + // intermediately nested path returns empty sequence + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[]', + 'lax $' AS "root_path" + COLUMNS( + a varchar(1) PATH 'lax "A"', + NESTED PATH 'lax $[*]' AS "nested_path_1" + COLUMNS ( + b varchar(1) PATH 'lax "B"', + NESTED PATH 'lax $' AS "nested_path_2" + COLUMNS( + c varchar(1) PATH 'lax "C"'))) + PLAN ("root_path" OUTER ("nested_path_1" INNER "nested_path_2"))) + """)) + .matches("VALUES ('A', CAST(null AS varchar(1)), CAST(null AS varchar(1)))"); + } + + @Test + public void testSiblingsRelationship() + { + // each sibling produces 1 row + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[]', + 'lax $' AS "root_path" + COLUMNS( + a varchar(1) PATH 'lax "A"', + NESTED PATH 'lax $' AS "nested_path_b" + COLUMNS (b varchar(1) PATH 'lax "B"'), + NESTED PATH 'lax $' AS "nested_path_c" + COLUMNS (c varchar(1) PATH 'lax "C"'), + NESTED PATH 'lax $' AS "nested_path_d" + COLUMNS (d varchar(1) PATH 'lax "D"')) + PLAN ("root_path" INNER ("nested_path_c" UNION ("nested_path_d" CROSS "nested_path_b")))) + """)) + .matches(""" + VALUES + ('A', CAST(null AS varchar(1)), 'C', CAST(null AS varchar(1))), + ('A', 'B', CAST(null AS varchar(1)), 'D') + """); + + // each sibling produces 2 rows + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[10, 1000]', + 'lax $' AS "root_path" + COLUMNS( + a varchar(1) PATH 'lax "A"', + NESTED PATH 'lax $[*]' AS "nested_path_1" + COLUMNS (b integer PATH 'lax $ * 1'), + NESTED PATH 'lax $[*]' AS "nested_path_2" + COLUMNS (c integer PATH 'lax $ * 2'), + NESTED PATH 'lax $[*]' AS "nested_path_3" + COLUMNS (d integer PATH 'lax $ * 3')) + PLAN ("root_path" INNER ("nested_path_2" UNION ("nested_path_3" CROSS "nested_path_1")))) + """)) + .matches(""" + VALUES + ('A', CAST(null AS integer), 20, CAST(null AS integer)), + ('A', null, 2000, null), + ('A', 10, null, 30), + ('A', 10, null, 3000), + ('A', 1000, null, 30), + ('A', 1000, null, 3000) + """); + + // one sibling produces empty result -- CROSS result is empty + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[10, 1000]', + 'lax $' AS "root_path" + COLUMNS( + a varchar(1) PATH 'lax "A"', + NESTED PATH 'lax $[*]' AS "nested_path_1" + COLUMNS (b integer PATH 'lax $ * 1'), + NESTED PATH 'lax $[42]' AS "nested_path_2" + COLUMNS (c integer PATH 'lax $ * 2')) + PLAN ("root_path" INNER ("nested_path_1" CROSS "nested_path_2"))) + """)) + .returnsEmptyResult(); + + // one sibling produces empty result -- UNION result contains the other sibling's result + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[10, 1000]', + 'lax $' AS "root_path" + COLUMNS( + a varchar(1) PATH 'lax "A"', + NESTED PATH 'lax $[*]' AS "nested_path_1" + COLUMNS (b integer PATH 'lax $ * 1'), + NESTED PATH 'lax $[42]' AS "nested_path_2" + COLUMNS (c integer PATH 'lax $ * 2')) + PLAN ("root_path" INNER ("nested_path_1" UNION "nested_path_2"))) + """)) + .matches(""" + VALUES + ('A', 10, CAST(null AS integer)), + ('A', 1000, null) + """); + } + + @Test + public void testImplicitColumnPath() + { + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '{"A" : 42, "b" : true}', + 'lax $' + COLUMNS( + a integer, + "b" boolean)) + """)) + .matches("VALUES (42, true)"); + + // the implicit column path is 'lax $.C'. It produces empty sequence, so the ON EMPTY clause determines the result + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '{"A" : 42, "b" : true}', + 'lax $' + COLUMNS(c varchar (5) DEFAULT 'empty' ON EMPTY DEFAULT 'error' ON ERROR)) + """)) + .matches("VALUES 'empty'"); + } + + @Test + public void testRootPathErrorHandling() + { + // error during root path evaluation handled according to top level EMPTY ON ERROR clause + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[]', + 'strict $[42]' + COLUMNS(a integer PATH 'lax 1') + EMPTY ON ERROR) + """)) + .returnsEmptyResult(); + + // error during root path evaluation handled according to top level ON ERROR clause which defaults to EMPTY ON ERROR + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[]', + 'strict $[42]' + COLUMNS(a integer PATH 'lax 1')) + """)) + .returnsEmptyResult(); + + // error during root path evaluation handled according to top level ERROR ON ERROR clause + assertTrinoExceptionThrownBy(() -> assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[]', + 'strict $[42]' + COLUMNS(a integer PATH 'lax 1') + ERROR ON ERROR) + """)) + .hasErrorCode(PATH_EVALUATION_ERROR) + .hasMessage("path evaluation failed: structural error: invalid array subscript for empty array"); + } + + @Test + public void testNestedPathErrorHandling() + { + // error during nested path evaluation handled according to top level EMPTY ON ERROR clause + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[]', + 'lax $' AS "root_path" + COLUMNS( + a integer PATH 'lax 1', + NESTED PATH 'strict $[42]' AS "nested_path" + COLUMNS(b integer PATH 'lax 2')) + PLAN DEFAULT(INNER) + EMPTY ON ERROR) + """)) + .returnsEmptyResult(); + + // error during nested path evaluation handled according to top level ON ERROR clause which defaults to EMPTY ON ERROR + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[]', + 'lax $' AS "root_path" + COLUMNS( + a integer PATH 'lax 1', + NESTED PATH 'strict $[42]' AS "nested_path" + COLUMNS(b integer PATH 'lax 2')) + PLAN DEFAULT(INNER)) + """)) + .returnsEmptyResult(); + + // error during nested path evaluation handled according to top level ERROR ON ERROR clause + assertTrinoExceptionThrownBy(() -> assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[]', + 'lax $' AS "root_path" + COLUMNS( + a integer PATH 'lax 1', + NESTED PATH 'strict $[42]' AS "nested_path" + COLUMNS(b integer PATH 'lax 2')) + PLAN DEFAULT(INNER) + ERROR ON ERROR) + """)) + .hasErrorCode(PATH_EVALUATION_ERROR) + .hasMessage("path evaluation failed: structural error: invalid array subscript for empty array"); + } + + @Test + public void testColumnPathErrorHandling() + { + // error during column path evaluation handled according to column's ERROR ON ERROR clause + assertTrinoExceptionThrownBy(() -> assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[]', + 'lax $' + COLUMNS(a integer PATH 'strict $[42]' ERROR ON ERROR) + EMPTY ON ERROR) + """)) + .hasErrorCode(PATH_EVALUATION_ERROR) + .hasMessage("path evaluation failed: structural error: invalid array subscript for empty array"); + + // error during column path evaluation handled according to column's ON ERROR clause which defaults to NULL ON ERROR + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[]', + 'lax $' + COLUMNS(a integer PATH 'strict $[42]') + EMPTY ON ERROR) + """)) + .matches("VALUES CAST(null as integer)"); + + // error during column path evaluation handled according to column's ON ERROR clause which defaults to ERROR ON ERROR because the top level error behavior is ERROR ON ERROR + assertTrinoExceptionThrownBy(() -> assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[]', + 'lax $' + COLUMNS(a integer PATH 'strict $[42]') + ERROR ON ERROR) + """)) + .hasErrorCode(PATH_EVALUATION_ERROR) + .hasMessage("path evaluation failed: structural error: invalid array subscript for empty array"); + } + + @Test + public void testEmptyInput() + { + assertThat(assertions.query(""" + SELECT * + FROM (SELECT '[]' WHERE rand() > 1) t(json_col), + JSON_TABLE( + json_col, + 'lax $' + COLUMNS(a integer PATH 'lax 1')) + """)) + .returnsEmptyResult(); + } + + @Test + public void testNullInput() + { + // if input is null, json_table returns empty result + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + CAST (null AS varchar), + 'lax $' + COLUMNS(a integer PATH 'lax 1')) + """)) + .returnsEmptyResult(); + + assertThat(assertions.query(""" + SELECT * + FROM (VALUES (CAST(null AS varchar)), (CAST(null AS varchar)), (CAST(null AS varchar))) t(json_col), + JSON_TABLE( + json_col, + 'lax $' + COLUMNS(a integer PATH 'lax 1')) + """)) + .returnsEmptyResult(); + + assertThat(assertions.query(""" + SELECT * + FROM (VALUES (CAST(null AS varchar)), (CAST(null AS varchar)), (CAST(null AS varchar))) t(json_col), + JSON_TABLE( + json_col, + 'lax $' + COLUMNS( + NESTED PATH 'lax $' + COLUMNS(a integer PATH 'lax 1'))) + """)) + .returnsEmptyResult(); + + // null as formatted input evaluates to empty sequence. json_table returns empty result + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + CAST (null AS varchar) FORMAT JSON, + 'lax $' + COLUMNS(a varchar FORMAT JSON PATH 'lax $')) + """)) + .returnsEmptyResult(); + } + + @Test + public void testNullPathParameter() + { + // null as SQL-value parameter "index" is evaluated to a JSON null, and causes type mismatch + assertTrinoExceptionThrownBy(() -> assertions.query(""" + SELECT * + FROM (SELECT '[1, 2, 3]', CAST(null AS integer)) t(json_col, index_col), + JSON_TABLE( + json_col, + 'lax $[$index]' PASSING index_col AS "index" + COLUMNS(a integer PATH 'lax 1') + ERROR ON ERROR) + """)) + .hasErrorCode(PATH_EVALUATION_ERROR) + .hasMessage("path evaluation failed: invalid item type. Expected: NUMBER, actual: NULL"); + + // null as JSON (formatted) parameter "index" evaluates to empty sequence, and causes type mismatch + assertTrinoExceptionThrownBy(() -> assertions.query(""" + SELECT * + FROM (SELECT '[1, 2, 3]', CAST(null AS varchar)) t(json_col, index_col), + JSON_TABLE( + json_col, + 'lax $[$index]' PASSING index_col FORMAT JSON AS "index" + COLUMNS(a integer PATH 'lax 1') + ERROR ON ERROR) + """)) + .hasErrorCode(PATH_EVALUATION_ERROR) + .hasMessage("path evaluation failed: array subscript 'from' value must be singleton numeric"); + } + + @Test + public void testNullDefaultValue() + { + assertThat(assertions.query(""" + SELECT a + FROM (SELECT null) t(empty_default), + JSON_TABLE( + '[1, 2, 3]', + 'lax $' + COLUMNS(a integer PATH 'lax $[42]' DEFAULT empty_default ON EMPTY DEFAULT -1 ON ERROR)) + """)) + .matches("VALUES CAST(null AS integer)"); + + assertThat(assertions.query(""" + SELECT a + FROM (SELECT null) t(error_default), + JSON_TABLE( + '[1, 2, 3]', + 'lax $' + COLUMNS(a integer PATH 'strict $[42]' DEFAULT -1 ON EMPTY DEFAULT error_default ON ERROR)) + """)) + .matches("VALUES CAST(null AS integer)"); + } + + @Test + public void testValueColumnCoercion() + { + // returned value cast to declared type + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' + COLUMNS(a real PATH 'lax $[last]')) + """)) + .matches("VALUES REAL '3'"); + + // default value cast to declared type + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' + COLUMNS(a real PATH 'lax $[42]' DEFAULT 42 ON EMPTY)) + """)) + .matches("VALUES REAL '42'"); + + // default ON EMPTY value is null. It is cast to declared type + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' + COLUMNS(a real PATH 'lax $[42]')) + """)) + .matches("VALUES CAST(null AS REAL)"); + + // default value cast to declared type + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' + COLUMNS(a real PATH 'strict $[42]' DEFAULT 42 ON ERROR)) + """)) + .matches("VALUES REAL '42'"); + + // default ON ERROR value is null. It is cast to declared type + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[1, 2, 3]', + 'lax $' + COLUMNS(a real PATH 'strict $[42]')) + """)) + .matches("VALUES CAST(null AS REAL)"); + } + + @Test + public void testQueryColumnFormat() + { + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[{"a" : true}]', + 'lax $' + COLUMNS(a varchar(50) FORMAT JSON PATH 'lax $[0]')) + """)) + .matches("VALUES CAST('{\"a\":true}' AS VARCHAR(50))"); + + String varbinaryLiteral = "X'" + base16().encode("{\"a\":true}".getBytes(UTF_16LE)) + "'"; + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[{"a" : true}]', + 'lax $' + COLUMNS(a varbinary FORMAT JSON ENCODING UTF16 PATH 'lax $[0]')) + """)) + .matches("VALUES " + varbinaryLiteral); + + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[{"a" : true}]', + 'lax $' + COLUMNS(a char(50) FORMAT JSON PATH 'lax $[42]' EMPTY OBJECT ON EMPTY)) + """)) + .matches("VALUES CAST('{}' AS CHAR(50))"); + + varbinaryLiteral = "X'" + base16().encode("[]".getBytes(UTF_16LE)) + "'"; + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[{"a" : true}]', + 'lax $' + COLUMNS(a varbinary FORMAT JSON ENCODING UTF16 PATH 'strict $[42]' EMPTY ARRAY ON ERROR)) + """)) + .matches("VALUES " + varbinaryLiteral); + + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '[{"a" : true}]', + 'lax $' + COLUMNS(a varbinary FORMAT JSON ENCODING UTF16 PATH 'lax $[42]' NULL ON EMPTY)) + """)) + .matches("VALUES CAST(null AS VARBINARY)"); + } + + @Test + public void testOrdinalityColumn() + { + assertThat(assertions.query(""" + SELECT * + FROM JSON_TABLE( + '["a", "b", "c", "d", "e", "f", "g", "h"]', + 'lax $[*]' AS "root_path" + COLUMNS( + o FOR ORDINALITY, + x varchar(1) PATH 'lax $')) + """)) + .matches(""" + VALUES + (BIGINT '1', 'a'), + (2, 'b'), + (3, 'c'), + (4, 'd'), + (5, 'e'), + (6, 'f'), + (7, 'g'), + (8, 'h') + """); + + assertThat(assertions.query(""" + SELECT * + FROM (VALUES + ('[["a", "b"], ["c", "d"], ["e", "f"]]'), + ('[["g", "h"], ["i", "j"], ["k", "l"]]')) t(json_col), + JSON_TABLE( + json_col, + 'lax $' AS "root_path" + COLUMNS( + o FOR ORDINALITY, + NESTED PATH 'lax $[0][*]' AS "nested_path_1" + COLUMNS ( + x1 varchar PATH 'lax $', + o1 FOR ORDINALITY), + NESTED PATH 'lax $[1][*]' AS "nested_path_2" + COLUMNS ( + x2 varchar PATH 'lax $', + o2 FOR ORDINALITY), + NESTED PATH 'lax $[2][*]' AS "nested_path_3" + COLUMNS ( + x3 varchar PATH 'lax $', + o3 FOR ORDINALITY)) + PLAN ("root_path" INNER ("nested_path_2" UNION ("nested_path_3" CROSS "nested_path_1")))) + """)) + .matches(""" + VALUES + ('[["a", "b"], ["c", "d"], ["e", "f"]]', BIGINT '1', VARCHAR 'a', BIGINT '1', CAST(null AS varchar), CAST(null AS bigint), VARCHAR 'e', BIGINT '1'), + ('[["a", "b"], ["c", "d"], ["e", "f"]]', 1, 'a', 1, null, null, 'f', 2), + ('[["a", "b"], ["c", "d"], ["e", "f"]]', 1, 'b', 2, null, null, 'e', 1), + ('[["a", "b"], ["c", "d"], ["e", "f"]]', 1, 'b', 2, null, null, 'f', 2), + ('[["a", "b"], ["c", "d"], ["e", "f"]]', 1, null, null, 'c', 1, null, null), + ('[["a", "b"], ["c", "d"], ["e", "f"]]', 1, null, null, 'd', 2, null, null), + + ('[["g", "h"], ["i", "j"], ["k", "l"]]', 1, VARCHAR 'g', BIGINT '1', CAST(null AS varchar), CAST(null AS bigint), VARCHAR 'k', BIGINT '1'), + ('[["g", "h"], ["i", "j"], ["k", "l"]]', 1, 'g', 1, null, null, 'l', 2), + ('[["g", "h"], ["i", "j"], ["k", "l"]]', 1, 'h', 2, null, null, 'k', 1), + ('[["g", "h"], ["i", "j"], ["k", "l"]]', 1, 'h', 2, null, null, 'l', 2), + ('[["g", "h"], ["i", "j"], ["k", "l"]]', 1, null, null, 'i', 1, null, null), + ('[["g", "h"], ["i", "j"], ["k", "l"]]', 1, null, null, 'j', 2, null, null) + """); + } +} diff --git a/core/trino-main/src/test/java/io/trino/type/TestJsonPath2016TypeSerialization.java b/core/trino-main/src/test/java/io/trino/type/TestJsonPath2016TypeSerialization.java index 7e28a1efc9eb..70cbf37a7d6c 100644 --- a/core/trino-main/src/test/java/io/trino/type/TestJsonPath2016TypeSerialization.java +++ b/core/trino-main/src/test/java/io/trino/type/TestJsonPath2016TypeSerialization.java @@ -71,7 +71,7 @@ public class TestJsonPath2016TypeSerialization { - private static final Type JSON_PATH_2016 = new JsonPath2016Type(new TypeDeserializer(TESTING_TYPE_MANAGER), new TestingBlockEncodingSerde()); + public static final Type JSON_PATH_2016 = new JsonPath2016Type(new TypeDeserializer(TESTING_TYPE_MANAGER), new TestingBlockEncodingSerde()); private static final RecursiveComparisonConfiguration COMPARISON_CONFIGURATION = RecursiveComparisonConfiguration.builder().withStrictTypeChecking(true).build(); @Test diff --git a/core/trino-parser/src/main/java/io/trino/sql/jsonpath/PathParser.java b/core/trino-parser/src/main/java/io/trino/sql/jsonpath/PathParser.java index e3be63349577..b15687e98550 100644 --- a/core/trino-parser/src/main/java/io/trino/sql/jsonpath/PathParser.java +++ b/core/trino-parser/src/main/java/io/trino/sql/jsonpath/PathParser.java @@ -40,13 +40,13 @@ public final class PathParser { private final BaseErrorListener errorListener; - public PathParser(Location startLocation) + public static PathParser withRelativeErrorLocation(Location startLocation) { requireNonNull(startLocation, "startLocation is null"); int pathStartLine = startLocation.line(); int pathStartColumn = startLocation.column(); - this.errorListener = new BaseErrorListener() + return new PathParser(new BaseErrorListener() { @Override public void syntaxError(Recognizer recognizer, Object offendingSymbol, int line, int charPositionInLine, String message, RecognitionException e) @@ -58,7 +58,26 @@ public void syntaxError(Recognizer recognizer, Object offendingSymbol, int int columnInQuery = line == 1 ? pathStartColumn + 1 + charPositionInLine : charPositionInLine + 1; throw new ParsingException(message, e, lineInQuery, columnInQuery); } - }; + }); + } + + public static PathParser withFixedErrorLocation(Location location) + { + requireNonNull(location, "location is null"); + + return new PathParser(new BaseErrorListener() + { + @Override + public void syntaxError(Recognizer recognizer, Object offendingSymbol, int line, int charPositionInLine, String message, RecognitionException e) + { + throw new ParsingException(message, e, location.line, location.column); + } + }); + } + + private PathParser(BaseErrorListener errorListener) + { + this.errorListener = requireNonNull(errorListener, "errorListener is null"); } public PathNode parseJsonPath(String path) diff --git a/core/trino-parser/src/test/java/io/trino/sql/jsonpath/TestPathParser.java b/core/trino-parser/src/test/java/io/trino/sql/jsonpath/TestPathParser.java index f294b2299d69..b645c75c0b3e 100644 --- a/core/trino-parser/src/test/java/io/trino/sql/jsonpath/TestPathParser.java +++ b/core/trino-parser/src/test/java/io/trino/sql/jsonpath/TestPathParser.java @@ -75,7 +75,7 @@ public class TestPathParser { - private static final PathParser PATH_PARSER = new PathParser(new Location(1, 0)); + private static final PathParser PATH_PARSER = PathParser.withRelativeErrorLocation(new Location(1, 0)); private static final RecursiveComparisonConfiguration COMPARISON_CONFIGURATION = RecursiveComparisonConfiguration.builder().withStrictTypeChecking(true).build(); @Test diff --git a/core/trino-spi/src/main/java/io/trino/spi/StandardErrorCode.java b/core/trino-spi/src/main/java/io/trino/spi/StandardErrorCode.java index 9500f3e4a66b..4751394a433d 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/StandardErrorCode.java +++ b/core/trino-spi/src/main/java/io/trino/spi/StandardErrorCode.java @@ -148,6 +148,9 @@ public enum StandardErrorCode INVALID_CATALOG_PROPERTY(124, USER_ERROR), CATALOG_UNAVAILABLE(125, USER_ERROR), MISSING_RETURN(126, USER_ERROR), + DUPLICATE_COLUMN_OR_PATH_NAME(127, USER_ERROR), + MISSING_PATH_NAME(128, USER_ERROR), + INVALID_PLAN(129, USER_ERROR), GENERIC_INTERNAL_ERROR(65536, INTERNAL_ERROR), TOO_MANY_REQUESTS_FAILED(65537, INTERNAL_ERROR),