Implemement distinct query support for complex JSON

src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs

@@ -303,7 +303,7 @@ public void ApplyDistinct()
                     case StructuralTypeProjectionExpression { StructuralType: IEntityType entityType } entityProjection
                         when entityType.IsMappedToJson():
                     {
-                        // For JSON entities, the identifier is the key that was generated when we convert from json to query root
+                        // For JSON owned entities, the identifier is the key that was generated when we convert from json to query root
                         // (OPENJSON, json_each, etc), but we can't use it for distinct, as it would warp the results.
                         // Instead, we will treat every non-key property as identifier.
 
@@ -334,8 +334,9 @@ when entityType.IsMappedToJson():
                     }
 
                     case StructuralTypeProjectionExpression { StructuralType: IComplexType } complexTypeProjection:
-                        // When distinct is applied to complex types, all properties - including ones in nested complex types - become
-                        // the identifier.
+                        // When distinct is applied to complex types, all columns - including ones in nested complex types - become
+                        // the identifier. Any JSON complex types found will simply have its single container column added like any
+                        // other column.
                         ProcessComplexType(complexTypeProjection);
 
                         void ProcessComplexType(StructuralTypeProjectionExpression complexTypeProjection)
@@ -350,11 +351,24 @@ void ProcessComplexType(StructuralTypeProjectionExpression complexTypeProjection
 
                             foreach (var complexProperty in complexType.GetComplexProperties())
                             {
-                                if (!complexProperty.IsCollection)
+                                switch (complexTypeProjection.BindComplexProperty(complexProperty))
                                 {
-                                    var complexPropertyShaper = (StructuralTypeShaperExpression)complexTypeProjection.BindComplexProperty(complexProperty);
-
-                                    ProcessComplexType((StructuralTypeProjectionExpression)complexPropertyShaper.ValueBufferExpression);
+                                    // Non-JSON non-collection type (table splitting): recurse inside and process all properties,
+                                    // as each property is mapped to its own column.
+                                    case StructuralTypeShaperExpression { ValueBufferExpression: StructuralTypeProjectionExpression projection }:
+                                        ProcessComplexType(projection);
+                                        continue;
+
+                                    // We have a JSON-mapped complex type.
+                                    // Ideally, we'd simply add the JSON container column to the identifiers list - just like for
+                                    // any regular property - but JSON columns aren't currently supported as identifiers (#36421).
+                                    case StructuralTypeShaperExpression { ValueBufferExpression: JsonQueryExpression }:
+                                    case CollectionResultExpression { QueryExpression: JsonQueryExpression }:
+                                        nonProcessableExpressionFound = true;
+                                        continue;
+
+                                    default:
+                                        throw new UnreachableException();
                                 }
                             }
                         }
@@ -381,8 +395,12 @@ void ProcessComplexType(StructuralTypeProjectionExpression complexTypeProjection
 
                         break;
 
-                    case JsonQueryExpression { StructuralType: IComplexType complexType } jsonQueryExpression:
-                        throw new NotImplementedException(); // #36296
+                    // We have a JSON-mapped complex type.
+                    // Ideally, we'd simply add the JSON container column to the identifiers list - just like for
+                    // any regular property - but JSON columns aren't currently supported as identifiers (#36421).
+                    case JsonQueryExpression jsonQueryExpression:
+                        nonProcessableExpressionFound = true;
+                        break;
 
                     case SqlExpression sqlExpression:
                         otherExpressions.Add(sqlExpression);
@@ -3755,25 +3773,27 @@ private SqlRemappingVisitor PushdownIntoSubqueryInternal(bool liftOrderings = tr
                     break;
                 }
 
-                if (item is StructuralTypeProjectionExpression projection)
-                {
-                    _clientProjections[i] = LiftStructuralProjectionFromSubquery(projection, subqueryAlias);
-                }
-                else if (item is JsonQueryExpression jsonQueryExpression)
+                switch (item)
                 {
-                    _clientProjections[i] = LiftJsonQueryFromSubquery(jsonQueryExpression);
-                }
-                else if (item is SqlExpression sqlExpression)
-                {
-                    var alias = _aliasForClientProjections[i];
-                    var outerColumn = subquery.GenerateOuterColumn(subqueryAlias, sqlExpression, alias);
-                    projectionMap[sqlExpression] = outerColumn;
-                    _clientProjections[i] = outerColumn;
-                    _aliasForClientProjections[i] = null;
-                }
-                else
-                {
-                    nestedQueryInProjection = true;
+                    case StructuralTypeProjectionExpression projection:
+                        _clientProjections[i] = LiftStructuralProjectionFromSubquery(projection, subqueryAlias);
+                        break;
+
+                    case JsonQueryExpression jsonQueryExpression:
+                        _clientProjections[i] = LiftJsonQueryFromSubquery(jsonQueryExpression);
+                        break;
+
+                    case SqlExpression sqlExpression:
+                        var alias = _aliasForClientProjections[i];
+                        var outerColumn = subquery.GenerateOuterColumn(subqueryAlias, sqlExpression, alias);
+                        projectionMap[sqlExpression] = outerColumn;
+                        _clientProjections[i] = outerColumn;
+                        _aliasForClientProjections[i] = null;
+                        break;
+
+                    default:
+                        nestedQueryInProjection = true;
+                        break;
                 }
             }
         }
@@ -3917,18 +3937,26 @@ StructuralTypeProjectionExpression LiftStructuralProjectionFromSubquery(
 
             foreach (var complexProperty in GetAllComplexPropertiesInHierarchy(projection.StructuralType))
             {
-                if (complexProperty.IsCollection)
+                switch (projection.BindComplexProperty(complexProperty))
                 {
-                    throw new NotImplementedException("#36296");
-                }
+                    // Non-JSON complex type - table splitting
+                    case StructuralTypeShaperExpression { ValueBufferExpression: StructuralTypeProjectionExpression p } shaper:
+                        complexPropertyCache[complexProperty] = shaper.Update(LiftStructuralProjectionFromSubquery(p, subqueryAlias));
+                        continue;
 
-                var complexPropertyShaper = (StructuralTypeShaperExpression)projection.BindComplexProperty(complexProperty);
+                    // JSON complex type (non-collection)
+                    case StructuralTypeShaperExpression { ValueBufferExpression: JsonQueryExpression jsonQuery } shaper:
+                        complexPropertyCache[complexProperty] = shaper.Update(LiftJsonQueryFromSubquery(jsonQuery));
+                        continue;
 
-                var complexTypeProjectionExpression = LiftStructuralProjectionFromSubquery(
-                    (StructuralTypeProjectionExpression)complexPropertyShaper.ValueBufferExpression,
-                    subqueryAlias);
+                    // JSON complex type (collection)
+                    case CollectionResultExpression { QueryExpression: JsonQueryExpression jsonQuery } shaper:
+                        complexPropertyCache[complexProperty] = shaper.Update(LiftJsonQueryFromSubquery(jsonQuery));
+                        continue;
 
-                complexPropertyCache[complexProperty] = complexPropertyShaper.Update(complexTypeProjectionExpression);
+                    default:
+                        throw new UnreachableException();
+                }
             }
 
             ColumnExpression? discriminatorExpression = null;

test/EFCore.Cosmos.FunctionalTests/Query/Relationships/OwnedNavigations/OwnedNavigationsCollectionCosmosTest.cs

@@ -57,6 +57,23 @@ FROM r IN c["RelatedCollection"]
 """);
     }
 
+
+    #region Distinct
+
+    public override Task Distinct()
+        => AssertTranslationFailed(base.Distinct);
+
+    public override Task Distinct_projected(QueryTrackingBehavior queryTrackingBehavior)
+        => Assert.ThrowsAnyAsync<Exception>(() => base.Distinct_projected(queryTrackingBehavior));
+
+    public override Task Distinct_over_projected_nested_collection()
+        => Assert.ThrowsAsync<InvalidOperationException>(base.Distinct_over_projected_nested_collection);
+
+    public override Task Distinct_over_projected_filtered_nested_collection()
+        => Assert.ThrowsAsync<InvalidOperationException>(base.Distinct_over_projected_nested_collection);
+
+    #endregion Distinct
+
     public override async Task Index_constant()
     {
         await base.Index_constant();

test/EFCore.Cosmos.FunctionalTests/Query/Relationships/OwnedNavigations/OwnedNavigationsSetOperationsCosmosTest.cs

@@ -0,0 +1,66 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace Microsoft.EntityFrameworkCore.Query.Relationships.OwnedNavigations;
+
+public class OwnedNavigationsSetOperationsCosmosTest : OwnedNavigationsSetOperationsTestBase<OwnedNavigationsCosmosFixture>
+{
+    public OwnedNavigationsSetOperationsCosmosTest(OwnedNavigationsCosmosFixture fixture, ITestOutputHelper testOutputHelper)
+        : base(fixture)
+    {
+        Fixture.TestSqlLoggerFactory.Clear();
+        Fixture.TestSqlLoggerFactory.SetTestOutputHelper(testOutputHelper);
+    }
+
+    public override async Task On_related()
+    {
+        await base.On_related();
+
+        AssertSql(
+            """
+SELECT VALUE c
+FROM root c
+WHERE (ARRAY_LENGTH(ARRAY_CONCAT(ARRAY(
+    SELECT VALUE r
+    FROM r IN c["RelatedCollection"]
+    WHERE (r["Int"] = 8)), ARRAY(
+    SELECT VALUE r0
+    FROM r0 IN c["RelatedCollection"]
+    WHERE (r0["String"] = "foo")))) = 4)
+""");
+    }
+
+    public override Task On_related_projected(QueryTrackingBehavior queryTrackingBehavior)
+        => Assert.ThrowsAsync<InvalidOperationException>(() => base.On_related_projected(queryTrackingBehavior));
+
+    public override Task On_related_Select_nested_with_aggregates(QueryTrackingBehavior queryTrackingBehavior)
+        => Assert.ThrowsAsync<InvalidOperationException>(() => base.On_related_projected(queryTrackingBehavior));
+
+    public override async Task On_nested()
+    {
+        await base.On_nested();
+
+        AssertSql(
+            """
+SELECT VALUE c
+FROM root c
+WHERE (ARRAY_LENGTH(ARRAY_CONCAT(ARRAY(
+    SELECT VALUE n
+    FROM n IN c["RequiredRelated"]["NestedCollection"]
+    WHERE (n["Int"] = 8)), ARRAY(
+    SELECT VALUE n0
+    FROM n0 IN c["RequiredRelated"]["NestedCollection"]
+    WHERE (n0["String"] = "foo")))) = 4)
+""");
+    }
+
+    public override Task Over_different_collection_properties()
+        => AssertTranslationFailed(base.Over_different_collection_properties);
+
+    [ConditionalFact]
+    public virtual void Check_all_tests_overridden()
+        => TestHelpers.AssertAllMethodsOverridden(GetType());
+
+    private void AssertSql(params string[] expected)
+        => Fixture.TestSqlLoggerFactory.AssertBaseline(expected);
+}

test/EFCore.Relational.Specification.Tests/Query/Relationships/ComplexJson/ComplexJsonCollectionRelationalTestBase.cs

@@ -15,6 +15,22 @@ public ComplexJsonCollectionRelationalTestBase(TFixture fixture, ITestOutputHelp
         Fixture.TestSqlLoggerFactory.SetTestOutputHelper(testOutputHelper);
     }
 
+    public override async Task Distinct_projected(QueryTrackingBehavior queryTrackingBehavior)
+    {
+        // #36421
+        var exception = await Assert.ThrowsAsync<InvalidOperationException>(
+            () => base.Distinct_projected(queryTrackingBehavior));
+
+        Assert.Equal(RelationalStrings.InsufficientInformationToIdentifyElementOfCollectionJoin, exception.Message);
+    }
+
+    public override async Task Distinct_over_projected_filtered_nested_collection()
+    {
+        var exception = await Assert.ThrowsAsync<InvalidOperationException>(base.Distinct_over_projected_filtered_nested_collection);
+
+        Assert.Equal(RelationalStrings.DistinctOnCollectionNotSupported, exception.Message);
+    }
+
     protected void AssertSql(params string[] expected)
         => Fixture.TestSqlLoggerFactory.AssertBaseline(expected);
 }

test/EFCore.Relational.Specification.Tests/Query/Relationships/Navigations/NavigationsCollectionRelationalTestBase.cs

@@ -13,6 +13,20 @@ public NavigationsCollectionRelationalTestBase(TFixture fixture, ITestOutputHelp
         Fixture.TestSqlLoggerFactory.SetTestOutputHelper(testOutputHelper);
     }
 
+    public override async Task Distinct_over_projected_nested_collection()
+    {
+        var exception = await Assert.ThrowsAsync<InvalidOperationException>(base.Distinct_over_projected_nested_collection);
+
+        Assert.Equal(RelationalStrings.DistinctOnCollectionNotSupported, exception.Message);
+    }
+
+    public override async Task Distinct_over_projected_filtered_nested_collection()
+    {
+        var exception = await Assert.ThrowsAsync<InvalidOperationException>(base.Distinct_over_projected_filtered_nested_collection);
+
+        Assert.Equal(RelationalStrings.DistinctOnCollectionNotSupported, exception.Message);
+    }
+
     protected void AssertSql(params string[] expected)
         => Fixture.TestSqlLoggerFactory.AssertBaseline(expected);
 }

test/EFCore.Relational.Specification.Tests/Query/Relationships/OwnedJson/OwnedJsonCollectionRelationalTestBase.cs

@@ -15,6 +15,20 @@ public OwnedJsonCollectionRelationalTestBase(TFixture fixture, ITestOutputHelper
         Fixture.TestSqlLoggerFactory.SetTestOutputHelper(testOutputHelper);
     }
 
+    public override async Task Distinct_over_projected_nested_collection()
+    {
+        var exception = await Assert.ThrowsAsync<InvalidOperationException>(base.Distinct_over_projected_nested_collection);
+
+        Assert.Equal(RelationalStrings.DistinctOnCollectionNotSupported, exception.Message);
+    }
+
+    public override async Task Distinct_over_projected_filtered_nested_collection()
+    {
+        var exception = await Assert.ThrowsAsync<InvalidOperationException>(base.Distinct_over_projected_filtered_nested_collection);
+
+        Assert.Equal(RelationalStrings.DistinctOnCollectionNotSupported, exception.Message);
+    }
+
     protected void AssertSql(params string[] expected)
         => Fixture.TestSqlLoggerFactory.AssertBaseline(expected);
 }

test/EFCore.Relational.Specification.Tests/Query/Relationships/OwnedNavigations/OwnedNavigationsCollectionRelationalTestBase.cs

@@ -13,6 +13,20 @@ public OwnedNavigationsCollectionRelationalTestBase(TFixture fixture, ITestOutpu
         Fixture.TestSqlLoggerFactory.SetTestOutputHelper(testOutputHelper);
     }
 
+    public override async Task Distinct_over_projected_nested_collection()
+    {
+        var exception = await Assert.ThrowsAsync<InvalidOperationException>(base.Distinct_over_projected_nested_collection);
+
+        Assert.Equal(RelationalStrings.DistinctOnCollectionNotSupported, exception.Message);
+    }
+
+    public override async Task Distinct_over_projected_filtered_nested_collection()
+    {
+        var exception = await Assert.ThrowsAsync<InvalidOperationException>(base.Distinct_over_projected_filtered_nested_collection);
+
+        Assert.Equal(RelationalStrings.DistinctOnCollectionNotSupported, exception.Message);
+    }
+
     public void AssertSql(params string[] expected)
         => Fixture.TestSqlLoggerFactory.AssertBaseline(expected);
 }

test/EFCore.Specification.Tests/Query/Relationships/ComplexProperties/ComplexPropertiesCollectionTestBase.cs

@@ -1,8 +1,6 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using Xunit.Sdk;
-
 namespace Microsoft.EntityFrameworkCore.Query.Relationships.ComplexProperties;
 
 public abstract class ComplexPropertiesCollectionTestBase<TFixture>(TFixture fixture)

test/EFCore.Specification.Tests/Query/Relationships/OwnedNavigations/OwnedNavigationsCollectionTestBase.cs

@@ -6,4 +6,20 @@ namespace Microsoft.EntityFrameworkCore.Query.Relationships.OwnedNavigations;
 public abstract class OwnedNavigationsCollectionTestBase<TFixture>(TFixture fixture) : RelationshipsCollectionTestBase<TFixture>(fixture)
         where TFixture : OwnedNavigationsFixtureBase, new()
 {
+    public override Task Distinct_projected(QueryTrackingBehavior queryTrackingBehavior)
+        => AssertOwnedTrackingQuery(queryTrackingBehavior, () => base.Distinct_projected(queryTrackingBehavior));
+
+    protected virtual async Task AssertOwnedTrackingQuery(QueryTrackingBehavior queryTrackingBehavior, Func<Task> test)
+    {
+        if (queryTrackingBehavior is QueryTrackingBehavior.TrackAll)
+        {
+            var message = (await Assert.ThrowsAsync<InvalidOperationException>(test)).Message;
+
+            Assert.Equal(CoreStrings.OwnedEntitiesCannotBeTrackedWithoutTheirOwner, message);
+
+            return;
+        }
+
+        await test();
+    }
 }

test/EFCore.Specification.Tests/Query/Relationships/RelationshipsCollectionTestBase.cs

@@ -26,6 +26,33 @@ public virtual Task OrderBy_ElementAt()
             ss => ss.Set<RootEntity>().Where(e => e.RelatedCollection.Count > 0
                 && e.RelatedCollection.OrderBy(r => r.Id).ElementAt(0).Int == 8));
 
+    #region Distinct
+
+    [ConditionalFact]
+    public virtual Task Distinct()
+        => AssertQuery(ss => ss.Set<RootEntity>().Where(e => e.RelatedCollection.Distinct().Count() == 2));
+
+    [ConditionalTheory]
+    [MemberData(nameof(TrackingData))]
+    public virtual Task Distinct_projected(QueryTrackingBehavior queryTrackingBehavior)
+        => AssertQuery(
+            ss => ss.Set<RootEntity>().OrderBy(e => e.Id).Select(e => e.RelatedCollection.Distinct().ToList()),
+            assertOrder: true,
+            elementAsserter: (e, a) => AssertCollection(e, a, elementSorter: r => r.Id),
+            queryTrackingBehavior: queryTrackingBehavior);
+
+    [ConditionalFact]
+    public virtual Task Distinct_over_projected_nested_collection()
+        => AssertQuery(ss => ss.Set<RootEntity>().Where(e =>
+            e.RelatedCollection.Select(r => r.NestedCollection).Distinct().Count() == 2));
+
+    [ConditionalFact]
+    public virtual Task Distinct_over_projected_filtered_nested_collection()
+        => AssertQuery(ss => ss.Set<RootEntity>().Where(e =>
+            e.RelatedCollection.Select(r => r.NestedCollection.Where(n => n.Int == 8)).Distinct().Count() == 2));
+
+    #endregion Distinct
+
     #region Index
 
     [ConditionalFact]