Analyze table function invocation in the context of a query

core/trino-main/src/main/java/io/trino/connector/ConnectorServices.java

@@ -38,6 +38,7 @@
 import io.trino.spi.procedure.Procedure;
 import io.trino.spi.ptf.ArgumentSpecification;
 import io.trino.spi.ptf.ConnectorTableFunction;
+import io.trino.spi.ptf.ReturnTypeSpecification.DescribedTable;
 import io.trino.spi.ptf.TableArgumentSpecification;
 import io.trino.spi.session.PropertyMetadata;
 import io.trino.split.RecordPageSourceProvider;
@@ -359,5 +360,9 @@ private static void validateTableFunction(ConnectorTableFunction tableFunction)
         // The 'keep when empty' or 'prune when empty' property must not be explicitly specified for a table argument with row semantics.
         // Such a table argument is implicitly 'prune when empty'. The TableArgumentSpecification.Builder enforces the 'prune when empty' property
         // for a table argument with row semantics.
+
+        if (tableFunction.getReturnTypeSpecification() instanceof DescribedTable describedTable) {
+            checkArgument(describedTable.getDescriptor().isTyped(), "field types missing in returned type specification");
+        }
     }
 }

core/trino-main/src/main/java/io/trino/sql/analyzer/Analysis.java

@@ -239,6 +239,8 @@ public class Analysis
     // names of tables and aliased relations. All names are resolved case-insensitive.
     private final Map<NodeRef<Relation>, QualifiedName> relationNames = new LinkedHashMap<>();
     private final Map<NodeRef<TableFunctionInvocation>, TableFunctionInvocationAnalysis> tableFunctionAnalyses = new LinkedHashMap<>();
+    private final Set<NodeRef<Relation>> aliasedRelations = new LinkedHashSet<>();
+    private final Set<NodeRef<TableFunctionInvocation>> polymorphicTableFunctions = new LinkedHashSet<>();
 
     public Analysis(@Nullable Statement root, Map<NodeRef<Parameter>, Expression> parameters, QueryType queryType)
     {
@@ -1232,6 +1234,26 @@ public QualifiedName getRelationName(Relation relation)
         return relationNames.get(NodeRef.of(relation));
     }
 
+    public void addAliased(Relation relation)
+    {
+        aliasedRelations.add(NodeRef.of(relation));
+    }
+
+    public boolean isAliased(Relation relation)
+    {
+        return aliasedRelations.contains(NodeRef.of(relation));
+    }
+
+    public void addPolymorphicTableFunction(TableFunctionInvocation invocation)
+    {
+        polymorphicTableFunctions.add(NodeRef.of(invocation));
+    }
+
+    public boolean isPolymorphicTableFunction(TableFunctionInvocation invocation)
+    {
+        return polymorphicTableFunctions.contains(NodeRef.of(invocation));
+    }
+
     private boolean isInputTable(Table table)
     {
         return !(isUpdateTarget(table) || isInsertTarget(table));
@@ -2192,6 +2214,7 @@ public static class TableFunctionInvocationAnalysis
         private final Map<String, Argument> arguments;
         private final List<TableArgumentAnalysis> tableArgumentAnalyses;
         private final List<List<String>> copartitioningLists;
+        private final int properColumnsCount;
         private final ConnectorTableFunctionHandle connectorTableFunctionHandle;
         private final ConnectorTransactionHandle transactionHandle;
 
@@ -2201,6 +2224,7 @@ public TableFunctionInvocationAnalysis(
                 Map<String, Argument> arguments,
                 List<TableArgumentAnalysis> tableArgumentAnalyses,
                 List<List<String>> copartitioningLists,
+                int properColumnsCount,
                 ConnectorTableFunctionHandle connectorTableFunctionHandle,
                 ConnectorTransactionHandle transactionHandle)
         {
@@ -2209,6 +2233,7 @@ public TableFunctionInvocationAnalysis(
             this.arguments = ImmutableMap.copyOf(arguments);
             this.tableArgumentAnalyses = ImmutableList.copyOf(tableArgumentAnalyses);
             this.copartitioningLists = ImmutableList.copyOf(copartitioningLists);
+            this.properColumnsCount = properColumnsCount;
             this.connectorTableFunctionHandle = requireNonNull(connectorTableFunctionHandle, "connectorTableFunctionHandle is null");
             this.transactionHandle = requireNonNull(transactionHandle, "transactionHandle is null");
         }
@@ -2238,6 +2263,16 @@ public List<List<String>> getCopartitioningLists()
             return copartitioningLists;
         }
 
+        /**
+         * Proper columns are the columns produced by the table function, as opposed to pass-through columns from input tables.
+         * Proper columns should be considered the actual result of the table function.
+         * @return the number of table function's proper columns
+         */
+        public int getProperColumnsCount()
+        {
+            return properColumnsCount;
+        }
+
         public ConnectorTableFunctionHandle getConnectorTableFunctionHandle()
         {
             return connectorTableFunctionHandle;

core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java

@@ -286,6 +286,7 @@
 import static io.trino.spi.StandardErrorCode.DUPLICATE_COLUMN_NAME;
 import static io.trino.spi.StandardErrorCode.DUPLICATE_NAMED_QUERY;
 import static io.trino.spi.StandardErrorCode.DUPLICATE_PROPERTY;
+import static io.trino.spi.StandardErrorCode.DUPLICATE_RANGE_VARIABLE;
 import static io.trino.spi.StandardErrorCode.DUPLICATE_WINDOW_NAME;
 import static io.trino.spi.StandardErrorCode.EXPRESSION_NOT_CONSTANT;
 import static io.trino.spi.StandardErrorCode.EXPRESSION_NOT_IN_DISTINCT;
@@ -300,6 +301,7 @@
 import static io.trino.spi.StandardErrorCode.INVALID_PARTITION_BY;
 import static io.trino.spi.StandardErrorCode.INVALID_RECURSIVE_REFERENCE;
 import static io.trino.spi.StandardErrorCode.INVALID_ROW_FILTER;
+import static io.trino.spi.StandardErrorCode.INVALID_TABLE_FUNCTION_INVOCATION;
 import static io.trino.spi.StandardErrorCode.INVALID_VIEW;
 import static io.trino.spi.StandardErrorCode.INVALID_WINDOW_FRAME;
 import static io.trino.spi.StandardErrorCode.INVALID_WINDOW_REFERENCE;
@@ -1519,47 +1521,62 @@ protected Scope visitTableFunctionInvocation(TableFunctionInvocation node, Optio
 
             List<List<String>> copartitioningLists = analyzeCopartitioning(node.getCopartitioning(), argumentsAnalysis.getTableArgumentAnalyses());
 
-            analysis.setTableFunctionAnalysis(node, new TableFunctionInvocationAnalysis(
-                    catalogHandle,
-                    function.getName(),
-                    argumentsAnalysis.getPassedArguments(),
-                    argumentsAnalysis.getTableArgumentAnalyses(),
-                    copartitioningLists,
-                    functionAnalysis.getHandle(),
-                    transactionHandle));
-
             // determine the result relation type.
             // The result relation type of a table function consists of:
             // 1. passed columns from input tables:
             // - for tables with the "pass through columns" option, these are all columns of the table,
             // - for tables without the "pass through columns" option, these are the partitioning columns of the table, if any.
             // 2. columns created by the table function, called the proper columns.
             ReturnTypeSpecification returnTypeSpecification = function.getReturnTypeSpecification();
+            if (returnTypeSpecification == GENERIC_TABLE || !argumentsAnalysis.getTableArgumentAnalyses().isEmpty()) {
+                analysis.addPolymorphicTableFunction(node);
+            }
             Optional<Descriptor> analyzedProperColumnsDescriptor = functionAnalysis.getReturnedType();
             Descriptor properColumnsDescriptor;
             if (returnTypeSpecification == ONLY_PASS_THROUGH) {
+                if (analysis.isAliased(node)) {
+                    // According to SQL standard ISO/IEC 9075-2, 7.6 <table reference>, p. 409,
+                    // table alias is prohibited for a table function with ONLY PASS THROUGH returned type.
+                    throw semanticException(INVALID_TABLE_FUNCTION_INVOCATION, node, "Alias specified for table function with ONLY PASS THROUGH return type");
+                }
                 // this option is only allowed if there are input tables
                 throw semanticException(NOT_SUPPORTED, node, "Returning only pass through columns is not yet supported for table functions");
             }
-            if (returnTypeSpecification == GENERIC_TABLE) {
+            else if (returnTypeSpecification == GENERIC_TABLE) {
+                // According to SQL standard ISO/IEC 9075-2, 7.6 <table reference>, p. 409,
+                // table alias is mandatory for a polymorphic table function invocation which produces proper columns.
+                // We don't enforce this requirement.
                 properColumnsDescriptor = analyzedProperColumnsDescriptor
                         .orElseThrow(() -> semanticException(MISSING_RETURN_TYPE, node, "Cannot determine returned relation type for table function " + node.getName()));
             }
-            else {
-                // returned type is statically declared at function declaration and cannot be overridden
+            else { // returned type is statically declared at function declaration
+                // According to SQL standard ISO/IEC 9075-2, 7.6 <table reference>, p. 409,
+                // table alias is mandatory for a polymorphic table function invocation which produces proper columns.
+                // We don't enforce this requirement.
+                // the declared type cannot be overridden
                 if (analyzedProperColumnsDescriptor.isPresent()) {
                     throw semanticException(AMBIGUOUS_RETURN_TYPE, node, "Returned relation type for table function %s is ambiguous", node.getName());
                 }
                 properColumnsDescriptor = ((DescribedTable) returnTypeSpecification).getDescriptor();
             }
 
             // currently we don't support input tables, so the output consists of proper columns only
-            // TODO implement SQL standard ISO/IEC 9075-2, 4.33 SQL-invoked routines, p. 123
+            // TODO implement SQL standard ISO/IEC 9075-2, 4.33 SQL-invoked routines, p. 123, 413, 414
             List<Field> fields = properColumnsDescriptor.getFields().stream()
                     // per spec, field names are mandatory
                     .map(field -> Field.newUnqualified(field.getName(), field.getType().orElseThrow(() -> new IllegalStateException("missing returned type for proper field"))))
                     .collect(toImmutableList());
 
+            analysis.setTableFunctionAnalysis(node, new TableFunctionInvocationAnalysis(
+                    catalogHandle,
+                    function.getName(),
+                    argumentsAnalysis.getPassedArguments(),
+                    argumentsAnalysis.getTableArgumentAnalyses(),
+                    copartitioningLists,
+                    properColumnsDescriptor.getFields().size(),
+                    functionAnalysis.getHandle(),
+                    transactionHandle));
+
             return createAndAssignScope(node, scope, fields);
         }
 
@@ -2367,6 +2384,9 @@ protected Scope visitPatternRecognitionRelation(PatternRecognitionRelation relat
         {
             Scope inputScope = process(relation.getInput(), scope);
 
+            // MATCH_RECOGNIZE cannot be applied to a polymorphic table function (SQL standard ISO/IEC 9075-2, 7.6 <table reference>, p. 409)
+            validateNoNestedTableFunction(relation.getInput(), "row pattern matching");
+
             // check that input table column names are not ambiguous
             // Note: This check is not compliant with SQL identifier semantics. Quoted identifiers should have different comparison rules than unquoted identifiers.
             // However, field names do not contain the information about quotation, and so every comparison is case-insensitive. For example, if there are fields named
@@ -2552,10 +2572,16 @@ private ExpressionAnalysis analyzePatternRecognitionExpression(Expression expres
         protected Scope visitAliasedRelation(AliasedRelation relation, Optional<Scope> scope)
         {
             analysis.setRelationName(relation, QualifiedName.of(ImmutableList.of(relation.getAlias())));
+            analysis.addAliased(relation.getRelation());
             Scope relationScope = process(relation.getRelation(), scope);
+            RelationType relationType = relationScope.getRelationType();
+
+            // special-handle table function invocation
+            if (relation.getRelation() instanceof TableFunctionInvocation function) {
+                return createAndAssignScope(relation, scope, aliasTableFunctionInvocation(relation, relationType, function));
+            }
 
             // todo this check should be inside of TupleDescriptor.withAlias, but the exception needs the node object
-            RelationType relationType = relationScope.getRelationType();
             if (relation.getColumnNames() != null) {
                 int totalColumns = relationType.getVisibleFieldCount();
                 if (totalColumns != relation.getColumnNames().size()) {
@@ -2588,6 +2614,83 @@ protected Scope visitAliasedRelation(AliasedRelation relation, Optional<Scope> s
             return createAndAssignScope(relation, scope, descriptor);
         }
 
+        // As described by the SQL standard ISO/IEC 9075-2, 7.6 <table reference>, p. 409
+        private RelationType aliasTableFunctionInvocation(AliasedRelation relation, RelationType relationType, TableFunctionInvocation function)
+        {
+            TableFunctionInvocationAnalysis tableFunctionAnalysis = analysis.getTableFunctionAnalysis(function);
+            int properColumnsCount = tableFunctionAnalysis.getProperColumnsCount();
+
+            // check that relation alias is different from range variables of all table arguments
+            tableFunctionAnalysis.getTableArgumentAnalyses().stream()
+                    .map(TableArgumentAnalysis::getName)
+                    .filter(Optional::isPresent)
+                    .map(Optional::get)
+                    .filter(name -> name.hasSuffix(QualifiedName.of(ImmutableList.of(relation.getAlias()))))
+                    .findFirst()
+                    .ifPresent(name -> {
+                        throw semanticException(DUPLICATE_RANGE_VARIABLE, relation.getAlias(), "Relation alias: %s is a duplicate of input table name: %s", relation.getAlias(), name);
+                    });
+
+            // build the new relation type. the alias must be applied to the proper columns only,
+            // and it must not shadow the range variables exposed by the table arguments
+            ImmutableList.Builder<Field> fieldsBuilder = ImmutableList.builder();
+            // first, put the table function's proper columns with alias
+            if (relation.getColumnNames() != null) {
+                // check that number of column aliases matches number of table function's proper columns
+                if (properColumnsCount != relation.getColumnNames().size()) {
+                    throw semanticException(MISMATCHED_COLUMN_ALIASES, relation, "Column alias list has %s entries but table function has %s proper columns", relation.getColumnNames().size(), properColumnsCount);
+                }
+                for (int i = 0; i < properColumnsCount; i++) {
+                    // proper columns are not hidden, so we don't need to skip hidden fields
+                    Field field = relationType.getFieldByIndex(i);
+                    fieldsBuilder.add(Field.newQualified(
+                            QualifiedName.of(ImmutableList.of(relation.getAlias())),
+                            Optional.of(relation.getColumnNames().get(i).getCanonicalValue()), // although the canonical name is recorded, fields are resolved case-insensitive
+                            field.getType(),
+                            field.isHidden(),
+                            field.getOriginTable(),
+                            field.getOriginColumnName(),
+                            field.isAliased()));
+                }
+            }
+            else {
+                for (int i = 0; i < properColumnsCount; i++) {
+                    Field field = relationType.getFieldByIndex(i);
+                    fieldsBuilder.add(Field.newQualified(
+                            QualifiedName.of(ImmutableList.of(relation.getAlias())),
+                            field.getName(),
+                            field.getType(),
+                            field.isHidden(),
+                            field.getOriginTable(),
+                            field.getOriginColumnName(),
+                            field.isAliased()));
+                }
+            }
+
+            // append remaining fields. They are not being aliased, so hidden fields are included
+            for (int i = properColumnsCount; i < relationType.getAllFieldCount(); i++) {
+                fieldsBuilder.add(relationType.getFieldByIndex(i));
+            }
+
+            List<Field> fields = fieldsBuilder.build();
+
+            // check that there are no duplicate names within the table function's proper columns
+            Set<String> names = new HashSet<>();
+            fields.subList(0, properColumnsCount).stream()
+                    .map(Field::getName)
+                    .filter(Optional::isPresent)
+                    .map(Optional::get)
+                    // field names are resolved case-insensitive
+                    .map(name -> name.toLowerCase(ENGLISH))
+                    .forEach(name -> {
+                        if (!names.add(name)) {
+                            throw semanticException(DUPLICATE_COLUMN_NAME, relation.getRelation(), "Duplicate name of table function proper column: " + name);
+                        }
+                    });
+
+            return new RelationType(fields);
+        }
+
         @Override
         protected Scope visitSampledRelation(SampledRelation relation, Optional<Scope> scope)
         {
@@ -2633,9 +2736,33 @@ protected Scope visitSampledRelation(SampledRelation relation, Optional<Scope> s
 
             analysis.setSampleRatio(relation, samplePercentageValue / 100);
             Scope relationScope = process(relation.getRelation(), scope);
+
+            // TABLESAMPLE cannot be applied to a polymorphic table function (SQL standard ISO/IEC 9075-2, 7.6 <table reference>, p. 409)
+            // Note: the below method finds a table function immediately nested in SampledRelation, or aliased.
+            // Potentially, a table function could be also nested with intervening PatternRecognitionRelation.
+            // Such case is handled in visitPatternRecognitionRelation().
+            validateNoNestedTableFunction(relation.getRelation(), "sample");
+
             return createAndAssignScope(relation, scope, relationScope.getRelationType());
         }
 
+        // this method should run after the `base` relation is processed, so that it is
+        // determined whether the table function is polymorphic
+        private void validateNoNestedTableFunction(Relation base, String context)
+        {
+            TableFunctionInvocation tableFunctionInvocation = null;
+            if (base instanceof TableFunctionInvocation invocation) {
+                tableFunctionInvocation = invocation;
+            }
+            else if (base instanceof AliasedRelation aliasedRelation &&
+                    aliasedRelation.getRelation() instanceof TableFunctionInvocation invocation) {
+                tableFunctionInvocation = invocation;
+            }
+            if (tableFunctionInvocation != null && analysis.isPolymorphicTableFunction(tableFunctionInvocation)) {
+                throw semanticException(INVALID_TABLE_FUNCTION_INVOCATION, base, "Cannot apply %s to polymorphic table function invocation", context);
+            }
+        }
+
         @Override
         protected Scope visitTableSubquery(TableSubquery node, Optional<Scope> scope)
         {