Support primitive collections

Closes dotnet#29427
Closes dotnet#30426
Closes dotnet#13617

Author: roji

All.sln.DotSettings

@@ -308,8 +308,10 @@ The .NET Foundation licenses this file to you under the MIT license.
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=navigations/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=niladic/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=NOCASE/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=OPENJSON/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=pluralizer/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Poolable/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=Postgre/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=pushdown/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=remapper/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=requiredness/@EntryIndexedValue">True</s:Boolean>

src/EFCore.Relational/Properties/RelationalStrings.Designer.cs

@@ -346,6 +346,9 @@
   <data name="EitherOfTwoValuesMustBeNull" xml:space="preserve">
     <value>Either {param1} or {param2} must be null.</value>
   </data>
+  <data name="EmptyCollectionNotSupportedAsConstantQueryRoot" xml:space="preserve">
+    <value>Empty constant collections are not supported as constant query roots.</value>
+  </data>
   <data name="EntityShortNameNotUnique" xml:space="preserve">
     <value>The short name for '{entityType1}' is '{discriminatorValue}' which is the same for '{entityType2}'. Every concrete entity type in the hierarchy must have a unique short name. Either rename one of the types or call modelBuilder.Entity&lt;TEntity&gt;().Metadata.SetDiscriminatorValue("NewShortName").</value>
   </data>

src/EFCore.Relational/Properties/RelationalStrings.resx

@@ -57,11 +57,10 @@ public ContainsTranslator(ISqlExpressionFactory sqlExpressionFactory)
     }
 
     private static bool ValidateValues(SqlExpression values)
-        => values is SqlConstantExpression || values is SqlParameterExpression;
+        => values is SqlConstantExpression or SqlParameterExpression;
 
     private static SqlExpression RemoveObjectConvert(SqlExpression expression)
-        => expression is SqlUnaryExpression sqlUnaryExpression
-            && sqlUnaryExpression.OperatorType == ExpressionType.Convert
+        => expression is SqlUnaryExpression { OperatorType: ExpressionType.Convert } sqlUnaryExpression
             && sqlUnaryExpression.Type == typeof(object)
                 ? sqlUnaryExpression.Operand
                 : expression;

src/EFCore.Relational/Query/Internal/ContainsTranslator.cs

@@ -1,7 +1,6 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using System.Runtime.CompilerServices;
 using Microsoft.EntityFrameworkCore.Query.SqlExpressions;
 using Microsoft.EntityFrameworkCore.Storage.Internal;
 
@@ -169,15 +168,17 @@ protected override Expression VisitSqlFragment(SqlFragmentExpression sqlFragment
     }
 
     private static bool IsNonComposedSetOperation(SelectExpression selectExpression)
-        => selectExpression.Offset == null
-            && selectExpression.Limit == null
-            && !selectExpression.IsDistinct
-            && selectExpression.Predicate == null
-            && selectExpression.Having == null
-            && selectExpression.Orderings.Count == 0
-            && selectExpression.GroupBy.Count == 0
-            && selectExpression.Tables.Count == 1
-            && selectExpression.Tables[0] is SetOperationBase setOperation
+        => selectExpression is
+            {
+                Tables: [SetOperationBase setOperation],
+                Offset: null,
+                Limit: null,
+                IsDistinct: false,
+                Predicate: null,
+                Having: null,
+                Orderings.Count: 0,
+                GroupBy.Count: 0
+            }
             && selectExpression.Projection.Count == setOperation.Source1.Projection.Count
             && selectExpression.Projection.Select(
                     (pe, index) => pe.Expression is ColumnExpression column
@@ -226,12 +227,7 @@ protected override Expression VisitSelect(SelectExpression selectExpression)
             subQueryIndent = _relationalCommandBuilder.Indent();
         }
 
-        if (IsNonComposedSetOperation(selectExpression))
-        {
-            // Naked set operation
-            GenerateSetOperation((SetOperationBase)selectExpression.Tables[0]);
-        }
-        else
+        if (!TryGenerateWithoutWrappingSelect(selectExpression))
         {
             _relationalCommandBuilder.Append("SELECT ");
 
@@ -300,6 +296,39 @@ protected override Expression VisitSelect(SelectExpression selectExpression)
         return selectExpression;
     }
 
+    /// <summary>
+    ///     If possible, generates the expression contained within the provided <paramref name="selectExpression" /> without the wrapping
+    ///     SELECT. This can be done for set operations and VALUES, which can appear as top-level statements without needing to be wrapped
+    ///     in SELECT.
+    /// </summary>
+    protected virtual bool TryGenerateWithoutWrappingSelect(SelectExpression selectExpression)
+    {
+        if (IsNonComposedSetOperation(selectExpression))
+        {
+            // Naked set operation
+            GenerateSetOperation((SetOperationBase)selectExpression.Tables[0]);
+            return true;
+        }
+
+        if (selectExpression is
+            {
+                Tables: [ValuesExpression valuesExpression],
+                Offset: null,
+                Limit: null,
+                IsDistinct: false,
+                Predicate: null,
+                Having: null,
+                Orderings.Count: 0,
+                GroupBy.Count: 0,
+            })
+        {
+            GenerateValues(valuesExpression, withParentheses: false);
+            return true;
+        }
+
+        return false;
+    }
+
     /// <summary>
     ///     Generates a pseudo FROM clause. Required by some providers when a query has no actual FROM clause.
     /// </summary>
@@ -312,9 +341,7 @@ protected virtual void GeneratePseudoFromClause()
     /// </summary>
     /// <param name="selectExpression">SelectExpression for which the empty projection will be generated.</param>
     protected virtual void GenerateEmptyProjection(SelectExpression selectExpression)
-    {
-        _relationalCommandBuilder.Append("1");
-    }
+        => _relationalCommandBuilder.Append("1");
 
     /// <inheritdoc />
     protected override Expression VisitProjection(ProjectionExpression projectionExpression)
@@ -371,16 +398,16 @@ protected override Expression VisitSqlFunction(SqlFunctionExpression sqlFunction
     /// <inheritdoc />
     protected override Expression VisitTableValuedFunction(TableValuedFunctionExpression tableValuedFunctionExpression)
     {
-        if (!string.IsNullOrEmpty(tableValuedFunctionExpression.StoreFunction.Schema))
+        if (!string.IsNullOrEmpty(tableValuedFunctionExpression.Schema))
         {
             _relationalCommandBuilder
-                .Append(_sqlGenerationHelper.DelimitIdentifier(tableValuedFunctionExpression.StoreFunction.Schema))
+                .Append(_sqlGenerationHelper.DelimitIdentifier(tableValuedFunctionExpression.Schema))
                 .Append(".");
         }
 
-        var name = tableValuedFunctionExpression.StoreFunction.IsBuiltIn
-            ? tableValuedFunctionExpression.StoreFunction.Name
-            : _sqlGenerationHelper.DelimitIdentifier(tableValuedFunctionExpression.StoreFunction.Name);
+        var name = tableValuedFunctionExpression.IsBuiltIn
+            ? tableValuedFunctionExpression.Name
+            : _sqlGenerationHelper.DelimitIdentifier(tableValuedFunctionExpression.Name);
 
         _relationalCommandBuilder
             .Append(name)
@@ -1132,6 +1159,28 @@ protected override Expression VisitRowNumber(RowNumberExpression rowNumberExpres
         return rowNumberExpression;
     }
 
+    /// <inheritdoc />
+    protected override Expression VisitRowValue(RowValueExpression rowValueExpression)
+    {
+        Sql.Append("(");
+
+        var values = rowValueExpression.Values;
+        var count = values.Count;
+        for (var i = 0; i < count; i++)
+        {
+            if (i > 0)
+            {
+                Sql.Append(", ");
+            }
+
+            Visit(values[i]);
+        }
+
+        Sql.Append(")");
+
+        return rowValueExpression;
+    }
+
     /// <summary>
     ///     Generates a set operation in the relational command.
     /// </summary>
@@ -1141,18 +1190,16 @@ protected virtual void GenerateSetOperation(SetOperationBase setOperation)
         GenerateSetOperationOperand(setOperation, setOperation.Source1);
         _relationalCommandBuilder
             .AppendLine()
-            .Append(GetSetOperation(setOperation))
+            .Append(
+                setOperation switch
+                {
+                    ExceptExpression => "EXCEPT",
+                    IntersectExpression => "INTERSECT",
+                    UnionExpression => "UNION",
+                    _ => throw new InvalidOperationException(CoreStrings.UnknownEntity("SetOperationType"))
+                })
             .AppendLine(setOperation.IsDistinct ? string.Empty : " ALL");
         GenerateSetOperationOperand(setOperation, setOperation.Source2);
-
-        static string GetSetOperation(SetOperationBase operation)
-            => operation switch
-            {
-                ExceptExpression => "EXCEPT",
-                IntersectExpression => "INTERSECT",
-                UnionExpression => "UNION",
-                _ => throw new InvalidOperationException(CoreStrings.UnknownEntity("SetOperationType"))
-            };
     }
 
     /// <summary>
@@ -1311,6 +1358,73 @@ void LiftPredicate(TableExpressionBase joinTable)
             RelationalStrings.ExecuteOperationWithUnsupportedOperatorInSqlGeneration(nameof(RelationalQueryableExtensions.ExecuteUpdate)));
     }
 
+    /// <inheritdoc />
+    protected override Expression VisitValues(ValuesExpression valuesExpression)
+    {
+        GenerateValues(valuesExpression, withParentheses: true);
+
+        return valuesExpression;
+    }
+
+    /// <summary>
+    ///     Generates a VALUES expression.
+    /// </summary>
+    protected virtual void GenerateValues(ValuesExpression valuesExpression, bool withParentheses = true)
+    {
+        // TODO: Review this!
+        if (withParentheses && valuesExpression.Alias is not null)
+        {
+            _relationalCommandBuilder.Append("(");
+        }
+
+        var rowValues = valuesExpression.RowValues;
+
+        // Some databases support providing the names of columns projected out of VALUES, e.g.
+        // SQL Server/PG: (VALUES (1, 3), (2, 4)) AS x(a, b). Others unfortunately don't; so by default, we extract out the first row,
+        // and generate a SELECT for it with the names, and a UNION ALL over the rest of the values.
+        _relationalCommandBuilder.Append("SELECT ");
+
+        Check.DebugAssert(rowValues.Count > 0, "rowValues.Count > 0");
+        var firstRowValues = rowValues[0].Values;
+        for (var i = 0; i < firstRowValues.Count; i++)
+        {
+            if (i > 0)
+            {
+                _relationalCommandBuilder.Append(", ");
+            }
+
+            Visit(firstRowValues[i]);
+
+            _relationalCommandBuilder
+                .Append(AliasSeparator)
+                .Append(valuesExpression.ColumnNames[i]);
+        }
+
+        if (rowValues.Count > 1)
+        {
+            _relationalCommandBuilder.Append(" UNION ALL VALUES ");
+
+            for (var i = 1; i < rowValues.Count; i++)
+            {
+                // TODO: Do we want newlines here?
+                if (i > 1)
+                {
+                    _relationalCommandBuilder.Append(", ");
+                }
+
+                Visit(valuesExpression.RowValues[i]);
+            }
+        }
+
+        if (withParentheses && valuesExpression.Alias is not null)
+        {
+            _relationalCommandBuilder
+                .Append(")")
+                .Append(AliasSeparator)
+                .Append(_sqlGenerationHelper.DelimitIdentifier(valuesExpression.Alias));
+        }
+    }
+
     /// <inheritdoc />
     protected override Expression VisitJsonScalar(JsonScalarExpression jsonScalarExpression)
         => throw new InvalidOperationException(

src/EFCore.Relational/Query/QuerySqlGenerator.cs

@@ -0,0 +1,82 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Microsoft.EntityFrameworkCore.Query.Internal;
+using Microsoft.EntityFrameworkCore.Query.SqlExpressions;
+
+namespace Microsoft.EntityFrameworkCore.Query;
+
+/// <inheritdoc />
+public class RelationalQueryRootProcessor : QueryRootProcessor
+{
+    private readonly ITypeMappingSource _typeMappingSource;
+    private readonly IModel _model;
+
+    /// <summary>
+    ///     Creates a new instance of the <see cref="RelationalQueryRootProcessor" /> class.
+    /// </summary>
+    /// <param name="typeMappingSource">The type mapping source.</param>
+    /// <param name="model">The model.</param>
+    public RelationalQueryRootProcessor(ITypeMappingSource typeMappingSource, IModel model)
+    {
+        _typeMappingSource = typeMappingSource;
+        _model = model;
+    }
+
+    /// <inheritdoc />
+    protected override Expression VisitMethodCall(MethodCallExpression methodCallExpression)
+    {
+        // Create query root node for table-valued functions
+        if (_model.FindDbFunction(methodCallExpression.Method) is { IsScalar: false, StoreFunction: var storeFunction })
+        {
+            // See issue #19970
+            return new TableValuedFunctionQueryRootExpression(
+                storeFunction.EntityTypeMappings.Single().EntityType,
+                storeFunction,
+                methodCallExpression.Arguments);
+        }
+
+        return base.VisitMethodCall(methodCallExpression);
+    }
+
+    /// <summary>
+    ///     Given a queryable constants over an element type which has a type mapping, converts it to a
+    ///     <see cref="ConstantQueryRootExpression" />; it will be translated to a SQL <see cref="ValuesExpression" />.
+    /// </summary>
+    /// <param name="constantExpression">The constant expression to attempt to convert to a query root.</param>
+    protected override Expression VisitQueryableConstant(ConstantExpression constantExpression)
+        => constantExpression.Type.TryGetSequenceType() is Type elementType && _typeMappingSource.FindMapping(elementType) is not null
+            ? new ConstantQueryRootExpression(constantExpression)
+            : constantExpression;
+
+    /// <inheritdoc />
+    protected override Expression VisitQueryableParameter(ParameterExpression parameterExpression)
+    {
+        // TODO: Decide whether this belongs here or in specific provider code. This means parameter query roots always get created
+        // (in enumerable/queryable context), but may not be translatable in the provider's QueryableMethodTranslatingEV. As long
+        // as we unwrap there, we *should* be OK, and so don't need an additional provider extension point here...
+
+        // TODO: Also, maybe this type checking should be in the base class.
+        // SQL Server's OpenJson, which we use to unpack the queryable parameter, does not support geometry (or any other non-built-in
+        // types)
+
+        // We convert to query roots only parameters whose CLR type has a collection type mapping (i.e. ElementTypeMapping isn't null).
+        // This allows the provider to determine exactly which types are supported as queryable collections (e.g. OPENJSON on SQL Server).
+        return _typeMappingSource.FindMapping(parameterExpression.Type) is { ElementTypeMapping: not null }
+                // ? new ParameterQueryRootExpression(elementTypeMapping.ClrType, parameterExpression)
+                ? new ParameterQueryRootExpression(parameterExpression.Type.GetSequenceType(), parameterExpression)
+                : parameterExpression;
+    }
+
+    /// <inheritdoc />
+    protected override Expression VisitExtension(Expression node)
+        => node switch
+        {
+            // We skip FromSqlQueryRootExpression, since that contains the arguments as an object array parameter, and don't want to convert
+            // that to a query root
+            FromSqlQueryRootExpression e => e,
+
+            _ => base.VisitExtension(node)
+        };
+
+}

src/EFCore.Relational/Query/RelationalQueryRootProcessor.cs

@@ -36,9 +36,13 @@ public override Expression NormalizeQueryableMethod(Expression expression)
     {
         expression = new RelationalQueryMetadataExtractingExpressionVisitor(_relationalQueryCompilationContext).Visit(expression);
         expression = base.NormalizeQueryableMethod(expression);
-        expression = new TableValuedFunctionToQueryRootConvertingExpressionVisitor(QueryCompilationContext.Model).Visit(expression);
         expression = new CollectionIndexerToElementAtNormalizingExpressionVisitor().Visit(expression);
 
         return expression;
     }
+
+    /// <inheritdoc />
+    public override Expression ProcessQueryRoots(Expression expression)
+        => new RelationalQueryRootProcessor(RelationalDependencies.RelationalTypeMappingSource, QueryCompilationContext.Model)
+            .Visit(expression);
 }