Work on inlined primitive collections (VALUES)

src/EFCore.Relational/Query/RelationalQueryableMethodTranslatingExpressionVisitor.cs

@@ -420,8 +420,7 @@ ParameterTranslationMode.Constant or ParameterTranslationMode.MultipleParameters
             var sqlExpression = sqlExpressions[i];
             rowExpressions[i] =
                 new RowValueExpression(
-                    new[]
-                    {
+                    [
                         // Since VALUES may not guarantee row ordering, we add an _ord value by which we'll order.
                         _sqlExpressionFactory.Constant(i, intTypeMapping),
                         // If no type mapping was inferred (i.e. no column in the inline collection), it's left null, to allow it to get
@@ -431,11 +430,11 @@ ParameterTranslationMode.Constant or ParameterTranslationMode.MultipleParameters
                         sqlExpression.TypeMapping is null && inferredTypeMaping is not null
                             ? _sqlExpressionFactory.ApplyTypeMapping(sqlExpression, inferredTypeMaping)
                             : sqlExpression
-                    });
+                    ]);
         }
 
         var alias = _sqlAliasManager.GenerateTableAlias("values");
-        var valuesExpression = new ValuesExpression(alias, rowExpressions, new[] { ValuesOrderingColumnName, ValuesValueColumnName });
+        var valuesExpression = new ValuesExpression(alias, rowExpressions, [ValuesOrderingColumnName, ValuesValueColumnName]);
 
         return CreateShapedQueryExpressionForValuesExpression(
             valuesExpression,

src/EFCore.Relational/Query/RelationalTypeMappingPostprocessor.cs

@@ -106,16 +106,7 @@ when TryGetInferredTypeMapping(columnExpression.TableAlias, columnExpression.Nam
             case ValuesExpression valuesExpression:
                 // By default, the ValuesExpression also contains an ordering by a synthetic increasing _ord. If the containing
                 // SelectExpression doesn't project it out or require it (limit/offset), strip that out.
-                // TODO: Strictly-speaking, stripping the ordering doesn't belong in this visitor which is about applying type mappings
-                return ApplyTypeMappingsOnValuesExpression(
-                    valuesExpression,
-                    stripOrdering: _currentSelectExpression is { Limit: null, Offset: null }
-                    && !_currentSelectExpression.Projection.Any(
-                        p => p.Expression is ColumnExpression
-                            {
-                                Name: RelationalQueryableMethodTranslatingExpressionVisitor.ValuesOrderingColumnName
-                            } c
-                            && c.TableAlias == valuesExpression.Alias));
+                return ApplyTypeMappingsOnValuesExpression(valuesExpression);
 
             // SqlExpressions without an inferred type mapping indicates a problem in EF - everything should have been inferred.
             // One exception is SqlFragmentExpression, which never has a type mapping.
@@ -135,8 +126,7 @@ when TryGetInferredTypeMapping(columnExpression.TableAlias, columnExpression.Nam
     ///     As an optimization, it can also strip the first _ord column if it's determined that it isn't needed (most cases).
     /// </summary>
     /// <param name="valuesExpression">The <see cref="ValuesExpression" /> to apply the mappings to.</param>
-    /// <param name="stripOrdering">Whether to strip the <c>_ord</c> column.</param>
-    protected virtual ValuesExpression ApplyTypeMappingsOnValuesExpression(ValuesExpression valuesExpression, bool stripOrdering)
+    protected virtual ValuesExpression ApplyTypeMappingsOnValuesExpression(ValuesExpression valuesExpression)
     {
         var inferredTypeMappings = TryGetInferredTypeMapping(
             valuesExpression.Alias, RelationalQueryableMethodTranslatingExpressionVisitor.ValuesValueColumnName, out var typeMapping)
@@ -146,9 +136,6 @@ protected virtual ValuesExpression ApplyTypeMappingsOnValuesExpression(ValuesExp
         Check.DebugAssert(
             valuesExpression.ColumnNames[0] == RelationalQueryableMethodTranslatingExpressionVisitor.ValuesOrderingColumnName,
             "First ValuesExpression column isn't the ordering column");
-        var newColumnNames = stripOrdering
-            ? valuesExpression.ColumnNames.Skip(1).ToArray()
-            : valuesExpression.ColumnNames;
 
         switch (valuesExpression)
         {
@@ -159,14 +146,9 @@ protected virtual ValuesExpression ApplyTypeMappingsOnValuesExpression(ValuesExp
                 for (var i = 0; i < newRowValues.Length; i++)
                 {
                     var rowValue = rowValues[i];
-                    var newValues = new SqlExpression[newColumnNames.Count];
+                    var newValues = new SqlExpression[valuesExpression.ColumnNames.Count];
                     for (var j = 0; j < valuesExpression.ColumnNames.Count; j++)
                     {
-                        if (j == 0 && stripOrdering)
-                        {
-                            continue;
-                        }
-
                         var value = rowValue.Values[j];
 
                         if (value.TypeMapping is null
@@ -182,13 +164,13 @@ protected virtual ValuesExpression ApplyTypeMappingsOnValuesExpression(ValuesExp
                             value = new SqlUnaryExpression(ExpressionType.Convert, value, value.Type, value.TypeMapping);
                         }
 
-                        newValues[j - (stripOrdering ? 1 : 0)] = value;
+                        newValues[j] = value;
                     }
 
                     newRowValues[i] = new RowValueExpression(newValues);
                 }
 
-                return new ValuesExpression(valuesExpression.Alias, newRowValues, null, newColumnNames);
+                return valuesExpression.Update(newRowValues);
             }
 
             // VALUES over a values parameter (i.e. a parameter representing the entire collection, that will be constantized into the SQL
@@ -203,10 +185,7 @@ protected virtual ValuesExpression ApplyTypeMappingsOnValuesExpression(ValuesExp
                     throw new UnreachableException("A RelationalTypeMapping collection type mapping could not be found");
                 }
 
-                return new ValuesExpression(
-                    valuesExpression.Alias,
-                    (SqlParameterExpression)valuesParameter.ApplyTypeMapping(collectionParameterTypeMapping),
-                    newColumnNames);
+                return valuesExpression.Update(valuesParameter.ApplyTypeMapping(collectionParameterTypeMapping));
             }
 
             default:

src/EFCore.Relational/Query/SqlExpressions/SqlParameterExpression.cs

@@ -74,8 +74,8 @@ public SqlParameterExpression(
     /// </summary>
     /// <param name="typeMapping">A relational type mapping to apply.</param>
     /// <returns>A new expression which has supplied type mapping.</returns>
-    public SqlExpression ApplyTypeMapping(RelationalTypeMapping? typeMapping)
-        => new SqlParameterExpression(InvariantName, Name, Type, IsNullable, TranslationMode, typeMapping);
+    public SqlParameterExpression ApplyTypeMapping(RelationalTypeMapping? typeMapping)
+        => new(InvariantName, Name, Type, IsNullable, TranslationMode, typeMapping);
 
     /// <inheritdoc />
     protected override Expression VisitChildren(ExpressionVisitor visitor)

src/EFCore.Relational/Query/SqlNullabilityProcessor.cs

@@ -122,7 +122,7 @@ protected override Expression VisitExtension(Expression node)
 
                 var intTypeMapping = (IntTypeMapping?)Dependencies.TypeMappingSource.FindMapping(typeof(int));
                 Check.DebugAssert(intTypeMapping is not null);
-                var valuesOrderingCounter = 1;
+                var valuesOrderingCounter = 0;
 
                 var processedValues = new List<RowValueExpression>();
 
@@ -151,13 +151,26 @@ protected override Expression VisitExtension(Expression node)
 
                     case ParameterTranslationMode.Constant:
                     {
-                        foreach (var value in values)
+                        for (var i = 0; i < values.Count; i++)
                         {
+                            var value = _sqlExpressionFactory.Constant(
+                                values[i],
+                                values[i]?.GetType() ?? typeof(object),
+                                sensitive: true,
+                                elementTypeMapping);
+
+                            // We currently add explicit conversions on the first row (but not to the _ord column), to ensure that the inferred
+                            // types are properly typed. See #30605 for removing that when not needed.
+                            if (i == 0)
+                            {
+                                value = new SqlUnaryExpression(ExpressionType.Convert, value, value.Type, value.TypeMapping);
+                            }
+
                             processedValues.Add(
                                 new RowValueExpression(
                                     ProcessValuesOrderingColumn(
                                         valuesExpression,
-                                        [_sqlExpressionFactory.Constant(value, value?.GetType() ?? typeof(object), sensitive: true, elementTypeMapping)],
+                                        [value],
                                         intTypeMapping,
                                         ref valuesOrderingCounter)));
                         }
@@ -1497,9 +1510,11 @@ protected virtual SqlExpression VisitJsonScalar(
         bool allowOptimizedExpansion,
         out bool nullable)
     {
-        nullable = jsonScalarExpression.IsNullable;
+        var json = Visit(jsonScalarExpression.Json, out var jsonNullable);
+
+        nullable = jsonNullable || jsonScalarExpression.IsNullable;
 
-        return jsonScalarExpression;
+        return jsonScalarExpression.Update(json);
     }
 
     /// <summary>

src/EFCore.Relational/Query/SqlTreePruner.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Collections;
 using Microsoft.EntityFrameworkCore.Query.SqlExpressions;
 
 namespace Microsoft.EntityFrameworkCore.Query;
@@ -117,6 +118,9 @@ protected override Expression VisitExtension(Expression node)
                     PruneSelect(source2, preserveProjection: true));
             }
 
+            case ValuesExpression values:
+                return PruneValues(values);
+
             default:
                 return base.VisitExtension(node);
         }
@@ -262,4 +266,117 @@ protected virtual SelectExpression PruneSelect(SelectExpression select, bool pre
         return select.Update(
             tables ?? select.Tables, predicate, groupBy, having, projections ?? select.Projection, orderings, offset, limit);
     }
+
+    /// <summary>
+    ///     Prunes a <see cref="ValuesExpression" />, removing columns inside it which aren't referenced.
+    ///     This currently removes the <c>_ord</c> column that gets added to preserve ordering, for cases where
+    ///     that ordering isn't actually necessary.
+    /// </summary>
+    /// <remarks>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </remarks>
+    [EntityFrameworkInternal]
+    protected virtual ValuesExpression PruneValues(ValuesExpression values)
+    {
+        BitArray? referencedColumns = null;
+        List<string>? newColumnNames = null;
+
+        if (ReferencedColumnMap.TryGetValue(values.Alias, out var referencedColumnNames))
+        {
+            // First, build a bitmap of which columns are referenced which we can efficiently access later
+            // as we traverse the rows. At the same time, build a list of the names of the referenced columns.
+            for (var i = 0; i < values.ColumnNames.Count; i++)
+            {
+                var columnName = values.ColumnNames[i];
+                var isColumnReferenced = referencedColumnNames.Contains(columnName);
+
+                if (newColumnNames is null && !isColumnReferenced)
+                {
+                    newColumnNames = new List<string>(values.ColumnNames.Count);
+                    referencedColumns = new BitArray(values.ColumnNames.Count);
+
+                    for (var j = 0; j < i; j++)
+                    {
+                        referencedColumns[j] = true;
+                        newColumnNames.Add(columnName);
+                    }
+                }
+
+                if (newColumnNames is not null)
+                {
+                    if (isColumnReferenced)
+                    {
+                        newColumnNames.Add(columnName);
+                    }
+
+                    referencedColumns![i] = isColumnReferenced;
+                }
+            }
+        }
+        else
+        {
+            // No columns were referenced at all on this ValuesExpression.
+            // This happens in some edge cases, e.g. there's a simple COUNT(*) over it (and so no specific columns are referenced).
+            // We can prune all columns but need to leave one, so that the ValuesExpression is still valid.
+            // Pick the first column, unless it happens to be the _ord column, in which case we pick the second one.
+            referencedColumns = new BitArray(values.ColumnNames.Count);
+            newColumnNames = new List<string>(1);
+
+            if (values.ColumnNames[0] is RelationalQueryableMethodTranslatingExpressionVisitor.ValuesOrderingColumnName)
+            {
+                referencedColumns[1] = true;
+                newColumnNames.Add(values.ColumnNames[1]);
+            }
+            else
+            {
+                referencedColumns[0] = true;
+                newColumnNames.Add(values.ColumnNames[0]);
+            }
+        }
+
+        if (referencedColumns is null)
+        {
+            return values;
+        }
+
+        // We know at least some columns are getting pruned.
+        Debug.Assert(newColumnNames is not null);
+
+        switch (values)
+        {
+            // If we have a value parameter (row values aren't specific in line), we still prune the column names.
+            // Later in SqlNullabilityProcessor, when the parameterized collection is inline to constants, we'll take
+            // the column names into account.
+            case ValuesExpression { ValuesParameter: not null }:
+                return new ValuesExpression(values.Alias, rowValues: null, values.ValuesParameter, newColumnNames);
+
+            // Go over the rows and create new ones without the pruned columns.
+            case ValuesExpression { RowValues: IReadOnlyList<RowValueExpression> rowValues }:
+                var newRowValues = new RowValueExpression[rowValues.Count];
+
+                for (var i = 0; i < rowValues.Count; i++)
+                {
+                    var oldValues = rowValues[i].Values;
+                    var newValues = new List<SqlExpression>(newColumnNames.Count);
+
+                    for (var j = 0; j < values.ColumnNames.Count; j++)
+                    {
+                        if (referencedColumns[j])
+                        {
+                            newValues.Add(oldValues[j]);
+                        }
+                    }
+
+                    newRowValues[i] = new RowValueExpression(newValues);
+                }
+
+                return new ValuesExpression(values.Alias, newRowValues, valuesParameter: null, newColumnNames);
+
+            default:
+                throw new UnreachableException();
+        }
+    }
 }

test/EFCore.Cosmos.FunctionalTests/Query/PrimitiveCollectionsQueryCosmosTest.cs

@@ -1020,6 +1020,13 @@ FROM root c
 """);
     }
 
+    public override async Task Inline_collection_index_Column_with_EF_Constant()
+    {
+        var exception = await Assert.ThrowsAsync<InvalidOperationException>(() => base.Inline_collection_index_Column_with_EF_Constant());
+
+        Assert.Equal(CoreStrings.EFConstantNotSupported, exception.Message);
+    }
+
     public override async Task Inline_collection_value_index_Column()
     {
         // Member indexer (c.Array[c.SomeMember]) isn't supported by Cosmos

test/EFCore.Specification.Tests/Query/PrimitiveCollectionsQueryTestBase.cs

@@ -680,6 +680,16 @@ public virtual Task Inline_collection_index_Column()
             ss => ss.Set<PrimitiveCollectionsEntity>().Where(c => new[] { 1, 2, 3 }[c.Int] == 1),
             ss => ss.Set<PrimitiveCollectionsEntity>().Where(c => (c.Int <= 2 ? new[] { 1, 2, 3 }[c.Int] : -1) == 1));
 
+    [ConditionalFact]
+    public virtual Task Inline_collection_index_Column_with_EF_Constant()
+    {
+        int[] ints = [1, 2, 3];
+
+        return AssertQuery(
+            ss => ss.Set<PrimitiveCollectionsEntity>().Where(c => EF.Constant(ints)[c.Int] == 1),
+            ss => ss.Set<PrimitiveCollectionsEntity>().Where(c => (c.Int <= 2 ? ints[c.Int] : -1) == 1));
+    }
+
     [ConditionalFact]
     public virtual Task Inline_collection_value_index_Column()
         => AssertQuery(

test/EFCore.SqlServer.FunctionalTests/Query/AdHocMiscellaneousQuerySqlServerTest.cs

@@ -2604,7 +2604,7 @@ WHERE [t].[Id] IN (?, ?, ?)
 FROM [TestEntities] AS [t]
 WHERE EXISTS (
     SELECT 1
-    FROM (VALUES (?), (?), (?)) AS [i]([Value])
+    FROM (VALUES (CAST(? AS int)), (?), (?)) AS [i]([Value])
     WHERE [i].[Value] = [t].[Id])
 """,
                 //
@@ -2628,7 +2628,7 @@ WHERE [t].[Id] IN (1, 2, 3)
 FROM [TestEntities] AS [t]
 WHERE EXISTS (
     SELECT 1
-    FROM (VALUES (1), (2), (3)) AS [i]([Value])
+    FROM (VALUES (CAST(1 AS int)), (2), (3)) AS [i]([Value])
     WHERE [i].[Value] = [t].[Id])
 """,
             //

test/EFCore.SqlServer.FunctionalTests/Query/GearsOfWarQuerySqlServerTest.cs

@@ -9194,7 +9194,7 @@ FROM [Weapons] AS [w]
     ) AS [w0]
     WHERE [w0].[row] <= ISNULL((
         SELECT [n].[Value]
-        FROM (VALUES (1, @numbers1), (2, @numbers2), (3, @numbers3)) AS [n]([_ord], [Value])
+        FROM (VALUES (@numbers1), (@numbers2), (@numbers3)) AS [n]([Value])
         ORDER BY [n].[Value]
         OFFSET 1 ROWS FETCH NEXT 1 ROWS ONLY), 0)
 ) AS [w1] ON [g].[FullName] = [w1].[OwnerFullName]

test/EFCore.SqlServer.FunctionalTests/Query/NonSharedPrimitiveCollectionsQuerySqlServerTest.cs

@@ -799,7 +799,7 @@ SELECT [t].[Id]
 FROM [TestEntity] AS [t]
 WHERE (
     SELECT COUNT(*)
-    FROM (VALUES (2), (999)) AS [i]([Value])
+    FROM (VALUES (CAST(2 AS int)), (999)) AS [i]([Value])
     WHERE [i].[Value] > [t].[Id]) = 1
 """);
                 break;
@@ -905,7 +905,7 @@ SELECT [t].[Id]
 FROM [TestEntity] AS [t]
 WHERE (
     SELECT COUNT(*)
-    FROM (VALUES (2), (999)) AS [i]([Value])
+    FROM (VALUES (CAST(2 AS int)), (999)) AS [i]([Value])
     WHERE [i].[Value] > [t].[Id]) = 1
 """);
     }