Support primitive collections

Closes dotnet#29427
Closes dotnet#30426
Closes dotnet#13617

Author: roji

All.sln.DotSettings

@@ -308,8 +308,10 @@ The .NET Foundation licenses this file to you under the MIT license.
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=navigations/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=niladic/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=NOCASE/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=OPENJSON/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=pluralizer/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Poolable/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=Postgre/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=pushdown/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=remapper/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=requiredness/@EntryIndexedValue">True</s:Boolean>

src/EFCore.Relational/Properties/RelationalStrings.Designer.cs

@@ -175,6 +175,9 @@
   <data name="ConflictingRowValuesSensitive" xml:space="preserve">
     <value>Instances of entity types '{firstEntityType}' and '{secondEntityType}' are mapped to the same row with the key value '{keyValue}', but have different property values '{firstConflictingValue}' and '{secondConflictingValue}' for the column '{column}'.</value>
   </data>
+  <data name="ConflictingTypeMappingsForPrimitiveCollection" xml:space="preserve">
+    <value>Store type '{storeType1}' was inferred for a primitive collection, but that primitive collection was previously inferred to have store type '{storeType2}'.</value>
+  </data>
   <data name="ConflictingSeedValues" xml:space="preserve">
     <value>A seed entity for entity type '{entityType}' has the same key value as another seed entity mapped to the same table '{table}', but have different values for the column '{column}'. Consider using 'DbContextOptionsBuilder.EnableSensitiveDataLogging' to see the conflicting values.</value>
   </data>
@@ -346,6 +349,9 @@
   <data name="EitherOfTwoValuesMustBeNull" xml:space="preserve">
     <value>Either {param1} or {param2} must be null.</value>
   </data>
+  <data name="EmptyCollectionNotSupportedAsConstantQueryRoot" xml:space="preserve">
+    <value>Empty constant collections are not supported as constant query roots.</value>
+  </data>
   <data name="EntityShortNameNotUnique" xml:space="preserve">
     <value>The short name for '{entityType1}' is '{discriminatorValue}' which is the same for '{entityType2}'. Every concrete entity type in the hierarchy must have a unique short name. Either rename one of the types or call modelBuilder.Entity&lt;TEntity&gt;().Metadata.SetDiscriminatorValue("NewShortName").</value>
   </data>

src/EFCore.Relational/Properties/RelationalStrings.resx

@@ -226,12 +226,7 @@ protected override Expression VisitSelect(SelectExpression selectExpression)
             subQueryIndent = _relationalCommandBuilder.Indent();
         }
 
-        if (IsNonComposedSetOperation(selectExpression))
-        {
-            // Naked set operation
-            GenerateSetOperation((SetOperationBase)selectExpression.Tables[0]);
-        }
-        else
+        if (!TryGenerateWithoutWrappingSelect(selectExpression))
         {
             _relationalCommandBuilder.Append("SELECT ");
 
@@ -300,6 +295,43 @@ protected override Expression VisitSelect(SelectExpression selectExpression)
         return selectExpression;
     }
 
+    /// <summary>
+    ///     If possible, generates the expression contained within the provided <paramref name="selectExpression" /> without the wrapping
+    ///     SELECT. This can be done for set operations and VALUES, which can appear as top-level statements without needing to be wrapped
+    ///     in SELECT.
+    /// </summary>
+    protected virtual bool TryGenerateWithoutWrappingSelect(SelectExpression selectExpression)
+    {
+        if (IsNonComposedSetOperation(selectExpression))
+        {
+            GenerateSetOperation((SetOperationBase)selectExpression.Tables[0]);
+            return true;
+        }
+
+        if (selectExpression is
+            {
+                Tables: [ValuesExpression valuesExpression],
+                Offset: null,
+                Limit: null,
+                IsDistinct: false,
+                Predicate: null,
+                Having: null,
+                Orderings.Count: 0,
+                GroupBy.Count: 0,
+            }
+            && selectExpression.Projection.Count == valuesExpression.ColumnNames.Count
+            && selectExpression.Projection.Select(
+                    (pe, index) => pe.Expression is ColumnExpression column
+                        && column.Name == valuesExpression.ColumnNames[index])
+                .All(e => e))
+        {
+            GenerateValues(valuesExpression);
+            return true;
+        }
+
+        return false;
+    }
+
     /// <summary>
     ///     Generates a pseudo FROM clause. Required by some providers when a query has no actual FROM clause.
     /// </summary>
@@ -371,16 +403,16 @@ protected override Expression VisitSqlFunction(SqlFunctionExpression sqlFunction
     /// <inheritdoc />
     protected override Expression VisitTableValuedFunction(TableValuedFunctionExpression tableValuedFunctionExpression)
     {
-        if (!string.IsNullOrEmpty(tableValuedFunctionExpression.StoreFunction.Schema))
+        if (!string.IsNullOrEmpty(tableValuedFunctionExpression.Schema))
         {
             _relationalCommandBuilder
-                .Append(_sqlGenerationHelper.DelimitIdentifier(tableValuedFunctionExpression.StoreFunction.Schema))
+                .Append(_sqlGenerationHelper.DelimitIdentifier(tableValuedFunctionExpression.Schema))
                 .Append(".");
         }
 
-        var name = tableValuedFunctionExpression.StoreFunction.IsBuiltIn
-            ? tableValuedFunctionExpression.StoreFunction.Name
-            : _sqlGenerationHelper.DelimitIdentifier(tableValuedFunctionExpression.StoreFunction.Name);
+        var name = tableValuedFunctionExpression.IsBuiltIn
+            ? tableValuedFunctionExpression.Name
+            : _sqlGenerationHelper.DelimitIdentifier(tableValuedFunctionExpression.Name);
 
         _relationalCommandBuilder
             .Append(name)
@@ -607,6 +639,7 @@ protected override Expression VisitSqlParameter(SqlParameterExpression sqlParame
     {
         var invariantName = sqlParameterExpression.Name;
         var parameterName = sqlParameterExpression.Name;
+        var typeMapping = sqlParameterExpression.TypeMapping!;
 
         // Try to see if a parameter already exists - if so, just integrate the same placeholder into the SQL instead of sending the same
         // data twice.
@@ -615,11 +648,10 @@ protected override Expression VisitSqlParameter(SqlParameterExpression sqlParame
         var parameter = _relationalCommandBuilder.Parameters.FirstOrDefault(
             p =>
                 p.InvariantName == parameterName
-                && p is TypeMappedRelationalParameter typeMappedRelationalParameter
-                && string.Equals(
-                    typeMappedRelationalParameter.RelationalTypeMapping.StoreType, sqlParameterExpression.TypeMapping!.StoreType,
-                    StringComparison.OrdinalIgnoreCase)
-                && typeMappedRelationalParameter.RelationalTypeMapping.Converter == sqlParameterExpression.TypeMapping!.Converter);
+                && p is TypeMappedRelationalParameter { RelationalTypeMapping: var existingTypeMapping }
+                && string.Equals(existingTypeMapping.StoreType, typeMapping.StoreType, StringComparison.OrdinalIgnoreCase)
+                && (existingTypeMapping.Converter is null && typeMapping.Converter is null
+                    || existingTypeMapping.Converter is not null && existingTypeMapping.Converter.Equals(typeMapping.Converter)));
 
         if (parameter is null)
         {
@@ -1132,6 +1164,28 @@ protected override Expression VisitRowNumber(RowNumberExpression rowNumberExpres
         return rowNumberExpression;
     }
 
+    /// <inheritdoc />
+    protected override Expression VisitRowValue(RowValueExpression rowValueExpression)
+    {
+        Sql.Append("(");
+
+        var values = rowValueExpression.Values;
+        var count = values.Count;
+        for (var i = 0; i < count; i++)
+        {
+            if (i > 0)
+            {
+                Sql.Append(", ");
+            }
+
+            Visit(values[i]);
+        }
+
+        Sql.Append(")");
+
+        return rowValueExpression;
+    }
+
     /// <summary>
     ///     Generates a set operation in the relational command.
     /// </summary>
@@ -1311,6 +1365,66 @@ void LiftPredicate(TableExpressionBase joinTable)
             RelationalStrings.ExecuteOperationWithUnsupportedOperatorInSqlGeneration(nameof(RelationalQueryableExtensions.ExecuteUpdate)));
     }
 
+    /// <inheritdoc />
+    protected override Expression VisitValues(ValuesExpression valuesExpression)
+    {
+        _relationalCommandBuilder.Append("(");
+
+        GenerateValues(valuesExpression);
+
+        _relationalCommandBuilder
+            .Append(")")
+            .Append(AliasSeparator)
+            .Append(_sqlGenerationHelper.DelimitIdentifier(valuesExpression.Alias));
+
+        return valuesExpression;
+    }
+
+    /// <summary>
+    ///     Generates a VALUES expression.
+    /// </summary>
+    protected virtual void GenerateValues(ValuesExpression valuesExpression)
+    {
+        var rowValues = valuesExpression.RowValues;
+
+        // Some databases support providing the names of columns projected out of VALUES, e.g.
+        // SQL Server/PG: (VALUES (1, 3), (2, 4)) AS x(a, b). Others unfortunately don't; so by default, we extract out the first row,
+        // and generate a SELECT for it with the names, and a UNION ALL over the rest of the values.
+        _relationalCommandBuilder.Append("SELECT ");
+
+        Check.DebugAssert(rowValues.Count > 0, "rowValues.Count > 0");
+        var firstRowValues = rowValues[0].Values;
+        for (var i = 0; i < firstRowValues.Count; i++)
+        {
+            if (i > 0)
+            {
+                _relationalCommandBuilder.Append(", ");
+            }
+
+            Visit(firstRowValues[i]);
+
+            _relationalCommandBuilder
+                .Append(AliasSeparator)
+                .Append(_sqlGenerationHelper.DelimitIdentifier(valuesExpression.ColumnNames[i]));
+        }
+
+        if (rowValues.Count > 1)
+        {
+            _relationalCommandBuilder.Append(" UNION ALL VALUES ");
+
+            for (var i = 1; i < rowValues.Count; i++)
+            {
+                // TODO: Do we want newlines here?
+                if (i > 1)
+                {
+                    _relationalCommandBuilder.Append(", ");
+                }
+
+                Visit(valuesExpression.RowValues[i]);
+            }
+        }
+    }
+
     /// <inheritdoc />
     protected override Expression VisitJsonScalar(JsonScalarExpression jsonScalarExpression)
         => throw new InvalidOperationException(

src/EFCore.Relational/Query/Internal/TableValuedFunctionToQueryRootConvertingExpressionVisitor.cs

@@ -0,0 +1,73 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using Microsoft.EntityFrameworkCore.Query.Internal;
+using Microsoft.EntityFrameworkCore.Query.SqlExpressions;
+
+namespace Microsoft.EntityFrameworkCore.Query;
+
+/// <inheritdoc />
+public class RelationalQueryRootProcessor : QueryRootProcessor
+{
+    private readonly ITypeMappingSource _typeMappingSource;
+    private readonly IModel _model;
+
+    /// <summary>
+    ///     Creates a new instance of the <see cref="RelationalQueryRootProcessor" /> class.
+    /// </summary>
+    /// <param name="typeMappingSource">The type mapping source.</param>
+    /// <param name="model">The model.</param>
+    public RelationalQueryRootProcessor(ITypeMappingSource typeMappingSource, IModel model)
+    {
+        _typeMappingSource = typeMappingSource;
+        _model = model;
+    }
+
+    /// <inheritdoc />
+    protected override Expression VisitMethodCall(MethodCallExpression methodCallExpression)
+    {
+        // Create query root node for table-valued functions
+        if (_model.FindDbFunction(methodCallExpression.Method) is { IsScalar: false, StoreFunction: var storeFunction })
+        {
+            // See issue #19970
+            return new TableValuedFunctionQueryRootExpression(
+                storeFunction.EntityTypeMappings.Single().EntityType,
+                storeFunction,
+                methodCallExpression.Arguments);
+        }
+
+        return base.VisitMethodCall(methodCallExpression);
+    }
+
+    /// <summary>
+    ///     Given a queryable constants over an element type which has a type mapping, converts it to a
+    ///     <see cref="ConstantQueryRootExpression" />; it will be translated to a SQL <see cref="ValuesExpression" />.
+    /// </summary>
+    /// <param name="constantExpression">The constant expression to attempt to convert to a query root.</param>
+    protected override Expression VisitQueryableConstant(ConstantExpression constantExpression)
+        // TODO: Note that we restrict to constants whose element type is mappable as-is. This excludes a constant list with an unsupported
+        // CLR type, where we could infer a type mapping with a value converter based on its later usage. However, converting all constant
+        // collections to query roots also carries risk.
+        => constantExpression.Type.TryGetSequenceType() is Type elementType && _typeMappingSource.FindMapping(elementType) is not null
+            ? new ConstantQueryRootExpression(constantExpression)
+            : constantExpression;
+
+    /// <inheritdoc />
+    protected override Expression VisitQueryableParameter(ParameterExpression parameterExpression)
+        // We convert to query roots only parameters whose CLR type has a collection type mapping (i.e. ElementTypeMapping isn't null).
+        // This allows the provider to determine exactly which types are supported as queryable collections (e.g. OPENJSON on SQL Server).
+        => _typeMappingSource.FindMapping(parameterExpression.Type) is { ElementTypeMapping: not null }
+            ? new ParameterQueryRootExpression(parameterExpression.Type.GetSequenceType(), parameterExpression)
+            : parameterExpression;
+
+    /// <inheritdoc />
+    protected override Expression VisitExtension(Expression node)
+        => node switch
+        {
+            // We skip FromSqlQueryRootExpression, since that contains the arguments as an object array parameter, and don't want to convert
+            // that to a query root
+            FromSqlQueryRootExpression e => e,
+
+            _ => base.VisitExtension(node)
+        };
+}