Fix to #30073 - JSON query regression in EF8 (#30086)

Problem was that when we were translating join, combining two SelectExpressions together, we were not remapping JSON column indexes stored in the inner select. This later result in shaper trying to read JSON column data from the wrong index and throwing the exception.

We already had a test that would have caught that, but it didn't return any data so the mismatch was never detected - fixed the data issue and added few more tests for good measure.

Fixes #30073

Author: maumar

src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs

@@ -1539,7 +1539,7 @@ Expression CopyProjectionToOuter(SelectExpression innerSelectExpression, Express
 
                         remappedConstant = Constant(
                             new JsonProjectionInfo(
-                                jsonProjectionInfo.JsonColumnIndex,
+                                projectionIndexMap[jsonProjectionInfo.JsonColumnIndex],
                                 newKeyAccessInfo,
                                 jsonProjectionInfo.AdditionalPath));
                     }

test/EFCore.Relational.Specification.Tests/Query/JsonQueryFixtureBase.cs

@@ -377,6 +377,7 @@ protected override void Seed(JsonQueryContext context)
     protected override void OnModelCreating(ModelBuilder modelBuilder, DbContext context)
     {
         modelBuilder.Entity<JsonEntityBasic>().Property(x => x.Id).ValueGeneratedNever();
+        modelBuilder.Entity<EntityBasic>().Property(x => x.Id).ValueGeneratedNever();
         modelBuilder.Entity<JsonEntityBasicForReference>().Property(x => x.Id).ValueGeneratedNever();
         modelBuilder.Entity<JsonEntityBasicForCollection>().Property(x => x.Id).ValueGeneratedNever();
         modelBuilder.Entity<JsonEntityBasic>().OwnsOne(

test/EFCore.Relational.Specification.Tests/Query/JsonQueryTestBase.cs

@@ -435,6 +435,59 @@ from e2 in g.DefaultIfEmpty()
             },
             entryCount: 40);
 
+    [ConditionalTheory]
+    [MemberData(nameof(IsAsyncData))]
+    public virtual Task Left_join_json_entities_json_being_inner(bool async)
+        => AssertQuery(
+            async,
+            ss => from e1 in ss.Set<JsonEntityBasic>()
+                  join e2 in ss.Set<JsonEntitySingleOwned>() on e1.Id equals e2.Id into g
+                  from e2 in g.DefaultIfEmpty()
+                  select new { e1, e2 },
+            elementSorter: e => (e.e1.Id, e.e2?.Id),
+            elementAsserter: (e, a) =>
+            {
+                AssertEqual(e.e1, a.e1);
+                AssertEqual(e.e2, a.e2);
+            },
+            entryCount: 44);
+
+    [ConditionalTheory]
+    [MemberData(nameof(IsAsyncData))]
+    public virtual Task Left_join_json_entities_complex_projection_json_being_inner(bool async)
+        => AssertQuery(
+            async,
+
+            ss => (from e1 in ss.Set<JsonEntityBasic>()
+                   join e2 in ss.Set<JsonEntitySingleOwned>() on e1.Id equals e2.Id into g
+                   from e2 in g.DefaultIfEmpty()
+                   select new
+                   {
+                       Id1 = e1.Id,
+                       Id2 = (int?)e2.Id,
+                       e1,
+                       e1.OwnedReferenceRoot,
+                       e1.OwnedReferenceRoot.OwnedReferenceBranch,
+                       e1.OwnedReferenceRoot.OwnedReferenceBranch.OwnedReferenceLeaf,
+                       e1.OwnedReferenceRoot.OwnedReferenceBranch.OwnedCollectionLeaf,
+                       e2,
+                       e2.Name
+                   }),
+            elementSorter: e => (e.Id1, e?.Id2),
+            elementAsserter: (e, a) =>
+            {
+                Assert.Equal(e.Id1, a.Id1);
+                Assert.Equal(e.Id2, a.Id2);
+                AssertEqual(e.e1, a.e1);
+                AssertEqual(e.OwnedReferenceRoot, a.OwnedReferenceRoot);
+                AssertEqual(e.OwnedReferenceBranch, a.OwnedReferenceBranch);
+                AssertEqual(e.OwnedReferenceLeaf, a.OwnedReferenceLeaf);
+                AssertCollection(e.OwnedCollectionLeaf, a.OwnedCollectionLeaf, ordered: true);
+                AssertEqual(e.e2, a.e2);
+                AssertEqual(e.Name, a.Name);
+            },
+            entryCount: 44);
+
     [ConditionalTheory]
     [MemberData(nameof(IsAsyncData))]
     public virtual Task Project_json_entity_FirstOrDefault_subquery(bool async)
@@ -1346,7 +1399,7 @@ public virtual Task Entity_including_collection_with_json(bool async)
             elementAsserter: (e, a) => AssertInclude(
                 e, a,
                 new ExpectedInclude<EntityBasic>(x => x.JsonEntityBasics)),
-            entryCount: 0);
+            entryCount: 41);
 
     [ConditionalTheory]
     [MemberData(nameof(IsAsyncData))]

test/EFCore.Relational.Specification.Tests/TestModels/JsonQuery/JsonQueryContext.cs

@@ -22,16 +22,18 @@ public JsonQueryContext(DbContextOptions options)
     public static void Seed(JsonQueryContext context)
     {
         var jsonEntitiesBasic = JsonQueryData.CreateJsonEntitiesBasic();
+        var entitiesBasic = JsonQueryData.CreateEntitiesBasic();
         var jsonEntitiesBasicForReference = JsonQueryData.CreateJsonEntitiesBasicForReference();
         var jsonEntitiesBasicForCollection = JsonQueryData.CreateJsonEntitiesBasicForCollection();
-        JsonQueryData.WireUp(jsonEntitiesBasic, jsonEntitiesBasicForReference, jsonEntitiesBasicForCollection);
+        JsonQueryData.WireUp(jsonEntitiesBasic, entitiesBasic, jsonEntitiesBasicForReference, jsonEntitiesBasicForCollection);
 
         var jsonEntitiesCustomNaming = JsonQueryData.CreateJsonEntitiesCustomNaming();
         var jsonEntitiesSingleOwned = JsonQueryData.CreateJsonEntitiesSingleOwned();
         var jsonEntitiesInheritance = JsonQueryData.CreateJsonEntitiesInheritance();
         var jsonEntitiesAllTypes = JsonQueryData.CreateJsonEntitiesAllTypes();
 
         context.JsonEntitiesBasic.AddRange(jsonEntitiesBasic);
+        context.EntitiesBasic.AddRange(entitiesBasic);
         context.JsonEntitiesBasicForReference.AddRange(jsonEntitiesBasicForReference);
         context.JsonEntitiesBasicForCollection.AddRange(jsonEntitiesBasicForCollection);
         context.JsonEntitiesCustomNaming.AddRange(jsonEntitiesCustomNaming);

test/EFCore.Relational.Specification.Tests/TestModels/JsonQuery/JsonQueryData.cs

@@ -7,11 +7,11 @@ public class JsonQueryData : ISetSource
 {
     public JsonQueryData()
     {
-        EntitiesBasic = new List<EntityBasic>();
         JsonEntitiesBasic = CreateJsonEntitiesBasic();
+        EntitiesBasic = CreateEntitiesBasic();
         JsonEntitiesBasicForReference = CreateJsonEntitiesBasicForReference();
         JsonEntitiesBasicForCollection = CreateJsonEntitiesBasicForCollection();
-        WireUp(JsonEntitiesBasic, JsonEntitiesBasicForReference, JsonEntitiesBasicForCollection);
+        WireUp(JsonEntitiesBasic, EntitiesBasic, JsonEntitiesBasicForReference, JsonEntitiesBasicForCollection);
 
         JsonEntitiesCustomNaming = CreateJsonEntitiesCustomNaming();
         JsonEntitiesSingleOwned = CreateJsonEntitiesSingleOwned();
@@ -292,6 +292,13 @@ public static IReadOnlyList<JsonEntityBasic> CreateJsonEntitiesBasic()
         return new List<JsonEntityBasic> { entity1 };
     }
 
+    public static IReadOnlyList<EntityBasic> CreateEntitiesBasic()
+    {
+        var entity1 = new EntityBasic { Id = 1, Name = "eb 1" };
+
+        return new List<EntityBasic> { entity1 };
+    }
+
     public static IReadOnlyList<JsonEntityBasicForReference> CreateJsonEntitiesBasicForReference()
     {
         var entity1 = new JsonEntityBasicForReference { Id = 1, Name = "EntityReference1" };
@@ -314,27 +321,30 @@ public static IReadOnlyList<JsonEntityBasicForCollection> CreateJsonEntitiesBasi
     }
 
     public static void WireUp(
-        IReadOnlyList<JsonEntityBasic> entitiesBasic,
+        IReadOnlyList<JsonEntityBasic> jsonEntitiesBasic,
+        IReadOnlyList<EntityBasic> entitiesBasic,
         IReadOnlyList<JsonEntityBasicForReference> entitiesBasicForReference,
         IReadOnlyList<JsonEntityBasicForCollection> entitiesBasicForCollection)
     {
-        entitiesBasic[0].EntityReference = entitiesBasicForReference[0];
-        entitiesBasicForReference[0].Parent = entitiesBasic[0];
-        entitiesBasicForReference[0].ParentId = entitiesBasic[0].Id;
+        entitiesBasic[0].JsonEntityBasics = new List<JsonEntityBasic> { jsonEntitiesBasic[0] };
+
+        jsonEntitiesBasic[0].EntityReference = entitiesBasicForReference[0];
+        entitiesBasicForReference[0].Parent = jsonEntitiesBasic[0];
+        entitiesBasicForReference[0].ParentId = jsonEntitiesBasic[0].Id;
 
-        entitiesBasic[0].EntityCollection = new List<JsonEntityBasicForCollection>
+        jsonEntitiesBasic[0].EntityCollection = new List<JsonEntityBasicForCollection>
         {
             entitiesBasicForCollection[0],
             entitiesBasicForCollection[1],
             entitiesBasicForCollection[2]
         };
 
-        entitiesBasicForCollection[0].Parent = entitiesBasic[0];
-        entitiesBasicForCollection[0].ParentId = entitiesBasic[0].Id;
-        entitiesBasicForCollection[1].Parent = entitiesBasic[0];
-        entitiesBasicForCollection[1].ParentId = entitiesBasic[0].Id;
-        entitiesBasicForCollection[2].Parent = entitiesBasic[0];
-        entitiesBasicForCollection[2].ParentId = entitiesBasic[0].Id;
+        entitiesBasicForCollection[0].Parent = jsonEntitiesBasic[0];
+        entitiesBasicForCollection[0].ParentId = jsonEntitiesBasic[0].Id;
+        entitiesBasicForCollection[1].Parent = jsonEntitiesBasic[0];
+        entitiesBasicForCollection[1].ParentId = jsonEntitiesBasic[0].Id;
+        entitiesBasicForCollection[2].Parent = jsonEntitiesBasic[0];
+        entitiesBasicForCollection[2].ParentId = jsonEntitiesBasic[0].Id;
     }
 
     public static IReadOnlyList<JsonEntityCustomNaming> CreateJsonEntitiesCustomNaming()
@@ -801,6 +811,16 @@ public IQueryable<TEntity> Set<TEntity>()
             return (IQueryable<TEntity>)JsonEntitiesAllTypes.OfType<JsonEntityAllTypes>().AsQueryable();
         }
 
+        if (typeof(TEntity) == typeof(JsonEntityBasicForReference))
+        {
+            return (IQueryable<TEntity>)JsonEntitiesBasicForReference.AsQueryable();
+        }
+
+        if (typeof(TEntity) == typeof(JsonEntityBasicForCollection))
+        {
+            return (IQueryable<TEntity>)JsonEntitiesBasicForCollection.AsQueryable();
+        }
+
         throw new InvalidOperationException("Invalid entity type: " + typeof(TEntity));
     }
 }

test/EFCore.SqlServer.FunctionalTests/Query/JsonQuerySqlServerTest.cs

@@ -435,6 +435,30 @@ FROM [JsonEntitiesSingleOwned] AS [j]
 """);
     }
 
+    public override async Task Left_join_json_entities_json_being_inner(bool async)
+    {
+        await base.Left_join_json_entities_json_being_inner(async);
+
+        AssertSql(
+"""
+SELECT [j].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot], [j0].[Id], [j0].[Name], [j0].[OwnedCollection]
+FROM [JsonEntitiesBasic] AS [j]
+LEFT JOIN [JsonEntitiesSingleOwned] AS [j0] ON [j].[Id] = [j0].[Id]
+""");
+    }
+
+    public override async Task Left_join_json_entities_complex_projection_json_being_inner(bool async)
+    {
+        await base.Left_join_json_entities_complex_projection_json_being_inner(async);
+
+        AssertSql(
+"""
+SELECT [j].[Id], [j0].[Id], [j].[EntityBasicId], [j].[Name], [j].[OwnedCollectionRoot], [j].[OwnedReferenceRoot], [j0].[Name], [j0].[OwnedCollection]
+FROM [JsonEntitiesBasic] AS [j]
+LEFT JOIN [JsonEntitiesSingleOwned] AS [j0] ON [j].[Id] = [j0].[Id]
+""");
+    }
+
     public override async Task Project_json_entity_FirstOrDefault_subquery(bool async)
     {
         await base.Project_json_entity_FirstOrDefault_subquery(async);