[rc2] Fix DefaultIfEmpty and nullability within SelectMany selector

src/EFCore.Relational/Query/RelationalQueryableMethodTranslatingExpressionVisitor.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System;
 using System.Diagnostics.CodeAnalysis;
 using Microsoft.EntityFrameworkCore.Query.Internal;
 using Microsoft.EntityFrameworkCore.Query.SqlExpressions;
@@ -235,6 +236,16 @@ [new FromSqlExpression(alias, sqlQueryRootExpression.Sql, sqlQueryRootExpression
     protected override Expression VisitMethodCall(MethodCallExpression methodCallExpression)
     {
         var method = methodCallExpression.Method;
+
+        if (method.DeclaringType == typeof(RelationalQueryableMethodTranslatingExpressionVisitor)
+            && method.IsGenericMethod
+            && method.GetGenericMethodDefinition() == _fakeDefaultIfEmptyMethodInfo.Value
+            && Visit(methodCallExpression.Arguments[0]) is ShapedQueryExpression source)
+        {
+            ((SelectExpression)source.QueryExpression).MakeProjectionNullable(_sqlExpressionFactory);
+            return source.UpdateShaperExpression(MarkShaperNullable(source.ShaperExpression));
+        }
+
         var translated = base.VisitMethodCall(methodCallExpression);
 
         // For Contains over a collection parameter, if the provider hasn't implemented TranslateCollection (e.g. OPENJSON on SQL
@@ -1214,7 +1225,10 @@ protected override Expression VisitMethodCall(MethodCallExpression methodCallExp
                 && methodCallExpression.Method.GetGenericMethodDefinition() == QueryableMethods.DefaultIfEmptyWithoutArgument)
             {
                 _defaultIfEmpty = true;
-                return Visit(methodCallExpression.Arguments[0]);
+
+                return Expression.Call(
+                    _fakeDefaultIfEmptyMethodInfo.Value.MakeGenericMethod(methodCallExpression.Method.GetGenericArguments()[0]),
+                    Visit(methodCallExpression.Arguments[0]));
             }
 
             if (!SupportsLiftingDefaultIfEmpty(methodCallExpression.Method))
@@ -2324,6 +2338,13 @@ private ShapedQueryExpression CreateShapedQueryExpressionForValuesExpression(
         return new ShapedQueryExpression(selectExpression, shaperExpression);
     }
 
+    private static IQueryable<TSource?> FakeDefaultIfEmpty<TSource>(IQueryable<TSource> source)
+        => throw new UnreachableException();
+
+    private static readonly Lazy<MethodInfo> _fakeDefaultIfEmptyMethodInfo = new(
+        () => typeof(RelationalQueryableMethodTranslatingExpressionVisitor)
+            .GetMethod(nameof(FakeDefaultIfEmpty), BindingFlags.NonPublic | BindingFlags.Static)!);
+
     /// <summary>
     ///     This visitor has been obsoleted; Extend RelationalTypeMappingPostprocessor instead, and invoke it from
     ///     <see cref="RelationalQueryTranslationPostprocessor.ProcessTypeMappings" />.

src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs

@@ -2527,7 +2527,20 @@ [new ProjectionExpression(nullSqlExpression, "empty")],
         _tables.Add(dummySelectExpression);
         _tables.Add(joinTable);
 
+        MakeProjectionNullable(sqlExpressionFactory);
+    }
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    [EntityFrameworkInternal]
+    public void MakeProjectionNullable(ISqlExpressionFactory sqlExpressionFactory)
+    {
         // Go over all projected columns and make them nullable; for non-nullable value types, add a SQL COALESCE as well.
+
         var projectionMapping = new Dictionary<ProjectionMember, Expression>();
         foreach (var (projectionMember, projection) in _projectionMapping)
         {

test/EFCore.Cosmos.FunctionalTests/Query/NorthwindMiscellaneousQueryCosmosTest.cs

@@ -3367,7 +3367,7 @@ public override async Task SelectMany_primitive_select_subquery(bool async)
         // Cosmos client evaluation. Issue #17246.
         Assert.Equal(
             CoreStrings.ExpressionParameterizationExceptionSensitive(
-                "value(Microsoft.EntityFrameworkCore.Query.NorthwindMiscellaneousQueryTestBase`1+<>c__DisplayClass175_0[Microsoft.EntityFrameworkCore.Query.NorthwindQueryCosmosFixture`1[Microsoft.EntityFrameworkCore.TestUtilities.NoopModelCustomizer]]).ss.Set().Any()"),
+                "value(Microsoft.EntityFrameworkCore.Query.NorthwindMiscellaneousQueryTestBase`1+<>c__DisplayClass177_0[Microsoft.EntityFrameworkCore.Query.NorthwindQueryCosmosFixture`1[Microsoft.EntityFrameworkCore.TestUtilities.NoopModelCustomizer]]).ss.Set().Any()"),
             (await Assert.ThrowsAsync<InvalidOperationException>(() => base.SelectMany_primitive_select_subquery(async))).Message);
 
         AssertSql();
@@ -3514,6 +3514,22 @@ public override async Task SelectMany_correlated_subquery_simple(bool async)
         AssertSql();
     }
 
+    public override async Task SelectMany_correlated_with_DefaultIfEmpty_and_Select_value_type_in_selector_throws(bool async)
+    {
+        // The test "passes" since the base implementation expects InvalidOperation (but for a different reason).
+        await base.SelectMany_correlated_with_DefaultIfEmpty_and_Select_value_type_in_selector_throws(async);
+
+        AssertSql();
+    }
+
+    public override async Task SelectMany_correlated_with_Select_value_type_and_DefaultIfEmpty_in_selector(bool async)
+    {
+        // Cosmos client evaluation. Issue #17246.
+        await AssertTranslationFailed(() => base.SelectMany_correlated_with_Select_value_type_and_DefaultIfEmpty_in_selector(async));
+
+        AssertSql();
+    }
+
     public override async Task SelectMany_correlated_subquery_hard(bool async)
     {
         // Cosmos client evaluation. Issue #17246.

test/EFCore.Specification.Tests/Query/NorthwindMiscellaneousQueryTestBase.cs

@@ -1848,6 +1848,38 @@ from e in ss.Set<Employee>().Where(e => e.City == c.City)
                 select new { c, e },
             assertOrder: true);
 
+    // DefaultIfEmpty() over empty set followed by Select() to a non-nullable type - should throw "Nullable object must have a value".
+    // (same happens for DefaultIfEmpty().Select() at the toplevel without SelectMany)
+    [ConditionalTheory, MemberData(nameof(IsAsyncData))] // #35950
+    public virtual Task SelectMany_correlated_with_DefaultIfEmpty_and_Select_value_type_in_selector_throws(bool async)
+        => Assert.ThrowsAsync<InvalidOperationException>(
+            () => AssertQuery(
+                async,
+                ss =>
+                    from c in ss.Set<Customer>()
+                    from o in c.Orders
+                        .Where(x => x.CustomerID == "NONEXISTENT") // Produce empty set for DefaultIfEmpty
+                        .DefaultIfEmpty()
+                        .Select(x => x.OrderID)
+                    select o));
+
+    // DefaultIfEmpty() after Select() to a non-nullable type - should add a COALESCE to the CLR default (0 here).
+    // Note that within the SelectMany selector, DIE is lifted out (and the INNER JOIN/CROSS APPLY is converted to
+    // LEFT JOIN/OUTER APPLY). But the COALESCE must still be applied.
+    [ConditionalTheory, MemberData(nameof(IsAsyncData))] // #35950
+    public virtual Task SelectMany_correlated_with_Select_value_type_and_DefaultIfEmpty_in_selector(bool async)
+        => AssertQuery(
+            async,
+            ss =>
+                from c in ss.Set<Customer>()
+                from o in c.Orders
+                    .Where(x => x.CustomerID == "NONEXISTENT") // Produce empty set for DefaultIfEmpty
+                    .Take(2)
+                    .OrderBy(x => true)
+                    .Select(x => x.OrderID)
+                    .DefaultIfEmpty()
+                select o);
+
     [ConditionalTheory, MemberData(nameof(IsAsyncData))]
     public virtual Task SelectMany_correlated_subquery_hard(bool async)
         => AssertQuery(

test/EFCore.SqlServer.FunctionalTests/Query/ComplexNavigationsQuerySqlServer160Test.cs

@@ -1808,10 +1808,10 @@ public override async Task SelectMany_with_nested_required_navigation_filter_and
 FROM [LevelOne] AS [l]
 LEFT JOIN [LevelTwo] AS [l0] ON [l].[Id] = [l0].[Level1_Required_Id]
 LEFT JOIN (
-    SELECT [l1].[Id], [l1].[OneToMany_Required_Inverse3Id]
+    SELECT [l1].[Id], [l1].[OneToMany_Required_Inverse3Id] AS [OneToMany_Required_Inverse3Id0]
     FROM [LevelThree] AS [l1]
     WHERE [l1].[Id] > 5
-) AS [l2] ON [l0].[Id] = [l2].[OneToMany_Required_Inverse3Id]
+) AS [l2] ON [l0].[Id] = [l2].[OneToMany_Required_Inverse3Id0]
 WHERE [l2].[Id] IS NOT NULL
 """);
     }
@@ -1988,14 +1988,14 @@ public override async Task SelectMany_with_navigation_filter_paging_and_explicit
 SELECT [l].[Id], [l].[Date], [l].[Name], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id]
 FROM [LevelOne] AS [l]
 LEFT JOIN (
-    SELECT [l1].[Id], [l1].[OneToMany_Required_Inverse2Id]
+    SELECT [l1].[Id], [l1].[OneToMany_Required_Inverse2Id0]
     FROM (
-        SELECT [l0].[Id], [l0].[OneToMany_Required_Inverse2Id], ROW_NUMBER() OVER(PARTITION BY [l0].[OneToMany_Required_Inverse2Id] ORDER BY [l0].[Id]) AS [row]
+        SELECT [l0].[Id], [l0].[OneToMany_Required_Inverse2Id] AS [OneToMany_Required_Inverse2Id0], ROW_NUMBER() OVER(PARTITION BY [l0].[OneToMany_Required_Inverse2Id] ORDER BY [l0].[Id]) AS [row]
         FROM [LevelTwo] AS [l0]
         WHERE [l0].[Id] > 5
     ) AS [l1]
     WHERE [l1].[row] <= 3
-) AS [l2] ON [l].[Id] = [l2].[OneToMany_Required_Inverse2Id]
+) AS [l2] ON [l].[Id] = [l2].[OneToMany_Required_Inverse2Id0]
 WHERE [l2].[Id] IS NOT NULL
 """);
     }

test/EFCore.SqlServer.FunctionalTests/Query/ComplexNavigationsQuerySqlServerTest.cs

@@ -1808,10 +1808,10 @@ public override async Task SelectMany_with_nested_required_navigation_filter_and
 FROM [LevelOne] AS [l]
 LEFT JOIN [LevelTwo] AS [l0] ON [l].[Id] = [l0].[Level1_Required_Id]
 LEFT JOIN (
-    SELECT [l1].[Id], [l1].[OneToMany_Required_Inverse3Id]
+    SELECT [l1].[Id], [l1].[OneToMany_Required_Inverse3Id] AS [OneToMany_Required_Inverse3Id0]
     FROM [LevelThree] AS [l1]
     WHERE [l1].[Id] > 5
-) AS [l2] ON [l0].[Id] = [l2].[OneToMany_Required_Inverse3Id]
+) AS [l2] ON [l0].[Id] = [l2].[OneToMany_Required_Inverse3Id0]
 WHERE [l2].[Id] IS NOT NULL
 """);
     }
@@ -1988,14 +1988,14 @@ public override async Task SelectMany_with_navigation_filter_paging_and_explicit
 SELECT [l].[Id], [l].[Date], [l].[Name], [l].[OneToMany_Optional_Self_Inverse1Id], [l].[OneToMany_Required_Self_Inverse1Id], [l].[OneToOne_Optional_Self1Id]
 FROM [LevelOne] AS [l]
 LEFT JOIN (
-    SELECT [l1].[Id], [l1].[OneToMany_Required_Inverse2Id]
+    SELECT [l1].[Id], [l1].[OneToMany_Required_Inverse2Id0]
     FROM (
-        SELECT [l0].[Id], [l0].[OneToMany_Required_Inverse2Id], ROW_NUMBER() OVER(PARTITION BY [l0].[OneToMany_Required_Inverse2Id] ORDER BY [l0].[Id]) AS [row]
+        SELECT [l0].[Id], [l0].[OneToMany_Required_Inverse2Id] AS [OneToMany_Required_Inverse2Id0], ROW_NUMBER() OVER(PARTITION BY [l0].[OneToMany_Required_Inverse2Id] ORDER BY [l0].[Id]) AS [row]
         FROM [LevelTwo] AS [l0]
         WHERE [l0].[Id] > 5
     ) AS [l1]
     WHERE [l1].[row] <= 3
-) AS [l2] ON [l].[Id] = [l2].[OneToMany_Required_Inverse2Id]
+) AS [l2] ON [l].[Id] = [l2].[OneToMany_Required_Inverse2Id0]
 WHERE [l2].[Id] IS NOT NULL
 """);
     }

test/EFCore.SqlServer.FunctionalTests/Query/ManyToManyNoTrackingQuerySqlServerTest.cs

@@ -457,10 +457,10 @@ public override async Task Select_many_over_skip_navigation_where(bool async)
 SELECT [s].[Id], [s].[CollectionInverseId], [s].[ExtraId], [s].[Name], [s].[ReferenceInverseId]
 FROM [EntityOnes] AS [e]
 LEFT JOIN (
-    SELECT [e0].[Id], [e0].[CollectionInverseId], [e0].[ExtraId], [e0].[Name], [e0].[ReferenceInverseId], [j].[OneId]
+    SELECT [e0].[Id], [e0].[CollectionInverseId], [e0].[ExtraId], [e0].[Name], [e0].[ReferenceInverseId], [j].[OneId] AS [OneId0]
     FROM [JoinOneToTwo] AS [j]
     INNER JOIN [EntityTwos] AS [e0] ON [j].[TwoId] = [e0].[Id]
-) AS [s] ON [e].[Id] = [s].[OneId]
+) AS [s] ON [e].[Id] = [s].[OneId0]
 """);
     }
 

test/EFCore.SqlServer.FunctionalTests/Query/ManyToManyQuerySqlServerTest.cs

@@ -456,10 +456,10 @@ public override async Task Select_many_over_skip_navigation_where(bool async)
 SELECT [s].[Id], [s].[CollectionInverseId], [s].[ExtraId], [s].[Name], [s].[ReferenceInverseId]
 FROM [EntityOnes] AS [e]
 LEFT JOIN (
-    SELECT [e0].[Id], [e0].[CollectionInverseId], [e0].[ExtraId], [e0].[Name], [e0].[ReferenceInverseId], [j].[OneId]
+    SELECT [e0].[Id], [e0].[CollectionInverseId], [e0].[ExtraId], [e0].[Name], [e0].[ReferenceInverseId], [j].[OneId] AS [OneId0]
     FROM [JoinOneToTwo] AS [j]
     INNER JOIN [EntityTwos] AS [e0] ON [j].[TwoId] = [e0].[Id]
-) AS [s] ON [e].[Id] = [s].[OneId]
+) AS [s] ON [e].[Id] = [s].[OneId0]
 """);
     }
 
@@ -1775,10 +1775,10 @@ public override async Task Select_many_over_skip_navigation_where_non_equality(b
 SELECT [s].[Id], [s].[CollectionInverseId], [s].[ExtraId], [s].[Name], [s].[ReferenceInverseId]
 FROM [EntityOnes] AS [e]
 LEFT JOIN (
-    SELECT [e0].[Id], [e0].[CollectionInverseId], [e0].[ExtraId], [e0].[Name], [e0].[ReferenceInverseId], [j].[OneId]
+    SELECT [e0].[Id], [e0].[CollectionInverseId], [e0].[ExtraId], [e0].[Name], [e0].[ReferenceInverseId], [j].[OneId] AS [OneId0], [e0].[Id] AS [Id0]
     FROM [JoinOneToTwo] AS [j]
     INNER JOIN [EntityTwos] AS [e0] ON [j].[TwoId] = [e0].[Id]
-) AS [s] ON [e].[Id] = [s].[OneId] AND [e].[Id] <> [s].[Id]
+) AS [s] ON [e].[Id] = [s].[OneId0] AND [e].[Id] <> [s].[Id0]
 """);
     }
 

test/EFCore.SqlServer.FunctionalTests/Query/NorthwindMiscellaneousQuerySqlServerTest.cs

@@ -6321,6 +6321,38 @@ FROM [Customers] AS [c]
 """);
     }
 
+    public override async Task SelectMany_correlated_with_DefaultIfEmpty_and_Select_value_type_in_selector_throws(bool async)
+    {
+        await base.SelectMany_correlated_with_DefaultIfEmpty_and_Select_value_type_in_selector_throws(async);
+
+        AssertSql(
+            """
+SELECT [o0].[OrderID]
+FROM [Customers] AS [c]
+LEFT JOIN (
+    SELECT [o].[OrderID], [o].[CustomerID]
+    FROM [Orders] AS [o]
+    WHERE [o].[CustomerID] = N'NONEXISTENT'
+) AS [o0] ON [c].[CustomerID] = [o0].[CustomerID]
+""");
+    }
+
+    public override async Task SelectMany_correlated_with_Select_value_type_and_DefaultIfEmpty_in_selector(bool async)
+    {
+        await base.SelectMany_correlated_with_Select_value_type_and_DefaultIfEmpty_in_selector(async);
+
+        AssertSql(
+            """
+SELECT COALESCE([o0].[OrderID], 0)
+FROM [Customers] AS [c]
+OUTER APPLY (
+    SELECT TOP(2) [o].[OrderID]
+    FROM [Orders] AS [o]
+    WHERE [c].[CustomerID] = [o].[CustomerID] AND [o].[CustomerID] = N'NONEXISTENT'
+) AS [o0]
+""");
+    }
+
     public override async Task Select_Property_when_shadow_unconstrained_generic_method(bool async)
     {
         await base.Select_Property_when_shadow_unconstrained_generic_method(async);

test/EFCore.SqlServer.FunctionalTests/Query/TPCManyToManyNoTrackingQuerySqlServerTest.cs

@@ -510,10 +510,10 @@ public override async Task Select_many_over_skip_navigation_where(bool async)
 SELECT [s].[Id], [s].[CollectionInverseId], [s].[ExtraId], [s].[Name], [s].[ReferenceInverseId]
 FROM [EntityOnes] AS [e]
 LEFT JOIN (
-    SELECT [e0].[Id], [e0].[CollectionInverseId], [e0].[ExtraId], [e0].[Name], [e0].[ReferenceInverseId], [j].[OneId]
+    SELECT [e0].[Id], [e0].[CollectionInverseId], [e0].[ExtraId], [e0].[Name], [e0].[ReferenceInverseId], [j].[OneId] AS [OneId0]
     FROM [JoinOneToTwo] AS [j]
     INNER JOIN [EntityTwos] AS [e0] ON [j].[TwoId] = [e0].[Id]
-) AS [s] ON [e].[Id] = [s].[OneId]
+) AS [s] ON [e].[Id] = [s].[OneId0]
 """);
     }
 
@@ -1986,10 +1986,10 @@ public override async Task Select_many_over_skip_navigation_where_non_equality(b
 SELECT [s].[Id], [s].[CollectionInverseId], [s].[ExtraId], [s].[Name], [s].[ReferenceInverseId]
 FROM [EntityOnes] AS [e]
 LEFT JOIN (
-    SELECT [e0].[Id], [e0].[CollectionInverseId], [e0].[ExtraId], [e0].[Name], [e0].[ReferenceInverseId], [j].[OneId]
+    SELECT [e0].[Id], [e0].[CollectionInverseId], [e0].[ExtraId], [e0].[Name], [e0].[ReferenceInverseId], [j].[OneId] AS [OneId0], [e0].[Id] AS [Id0]
     FROM [JoinOneToTwo] AS [j]
     INNER JOIN [EntityTwos] AS [e0] ON [j].[TwoId] = [e0].[Id]
-) AS [s] ON [e].[Id] = [s].[OneId] AND [e].[Id] <> [s].[Id]
+) AS [s] ON [e].[Id] = [s].[OneId0] AND [e].[Id] <> [s].[Id0]
 """);
     }
 
@@ -2342,10 +2342,10 @@ public override async Task Select_many_over_skip_navigation_where_unidirectional
 SELECT [s].[Id], [s].[CollectionInverseId], [s].[ExtraId], [s].[Name], [s].[ReferenceInverseId]
 FROM [UnidirectionalEntityOnes] AS [u]
 LEFT JOIN (
-    SELECT [u1].[Id], [u1].[CollectionInverseId], [u1].[ExtraId], [u1].[Name], [u1].[ReferenceInverseId], [u0].[OneId]
+    SELECT [u1].[Id], [u1].[CollectionInverseId], [u1].[ExtraId], [u1].[Name], [u1].[ReferenceInverseId], [u0].[OneId] AS [OneId0]
     FROM [UnidirectionalJoinOneToTwo] AS [u0]
     INNER JOIN [UnidirectionalEntityTwos] AS [u1] ON [u0].[TwoId] = [u1].[Id]
-) AS [s] ON [u].[Id] = [s].[OneId]
+) AS [s] ON [u].[Id] = [s].[OneId0]
 """);
     }
 
@@ -2712,10 +2712,10 @@ public override async Task Select_many_over_skip_navigation_where_non_equality_u
 SELECT [s].[Id], [s].[CollectionInverseId], [s].[ExtraId], [s].[Name], [s].[ReferenceInverseId]
 FROM [UnidirectionalEntityOnes] AS [u]
 LEFT JOIN (
-    SELECT [u1].[Id], [u1].[CollectionInverseId], [u1].[ExtraId], [u1].[Name], [u1].[ReferenceInverseId], [u0].[OneId]
+    SELECT [u1].[Id], [u1].[CollectionInverseId], [u1].[ExtraId], [u1].[Name], [u1].[ReferenceInverseId], [u0].[OneId] AS [OneId0], [u1].[Id] AS [Id0]
     FROM [UnidirectionalJoinOneToTwo] AS [u0]
     INNER JOIN [UnidirectionalEntityTwos] AS [u1] ON [u0].[TwoId] = [u1].[Id]
-) AS [s] ON [u].[Id] = [s].[OneId] AND [u].[Id] <> [s].[Id]
+) AS [s] ON [u].[Id] = [s].[OneId0] AND [u].[Id] <> [s].[Id0]
 """);
     }