Skip to content

Implemement set operation query support for complex JSON #36417

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
roji merged 3 commits into from
Jul 24, 2025
Merged

Implemement set operation query support for complex JSON #36417

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5ebed4f
Implemement set operation query support for complex JSON
roji Jul 22, 2025
25a9d0a
Fix nullability handling
roji Jul 24, 2025
5e7bbc0
Some testing additions
roji Jul 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,9 @@ paket-files/
.idea/
*.sln.iml

# Visual Studio Code
.vscode/

# CodeRush personal settings
.cr/personal

Expand Down
2 changes: 1 addition & 1 deletion src/EFCore.Relational/Query/RelationalSqlTranslatingExpressionVisitor.StructuralEquality.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,7 +380,7 @@ SqlExpression Process(Expression expression)
jsonQuery.JsonColumn.TypeMapping,
jsonQuery.IsNullable),

// As above, but for a complex JSON collectio
// As above, but for a complex JSON collection
CollectionResultExpression { QueryExpression: JsonQueryExpression jsonQuery }
=> new JsonScalarExpression(
jsonQuery.JsonColumn,
Expand Down
96 changes: 83 additions & 13 deletions src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2086,7 +2086,7 @@ private void ApplySetOperation(

var outerIdentifiers = select1._identifier.Count == select2._identifier.Count
? new ColumnExpression?[select1._identifier.Count]
: Array.Empty<ColumnExpression?>();
: [];
var entityProjectionIdentifiers = new List<ColumnExpression>();
var entityProjectionValueComparers = new List<ValueComparer>();
var otherExpressions = new List<(SqlExpression Expression, ValueComparer Comparer)>();
Expand Down Expand Up @@ -2333,22 +2333,91 @@ StructuralTypeProjectionExpression ProcessStructuralType(
var complexPropertyShaper1 = structuralProjection1.BindComplexProperty(complexProperty);
var complexPropertyShaper2 = structuralProjection2.BindComplexProperty(complexProperty);

if (complexPropertyShaper1 is not StructuralTypeShaperExpression nonCollectionShaper1
|| complexPropertyShaper2 is not StructuralTypeShaperExpression nonCollectionShaper2)
switch ((complexPropertyShaper1, complexPropertyShaper2))
{
throw new NotImplementedException("Set operation over collection complex properties");
// Set operation over type that contains a structural type mapped to table splitting -
// recurse to continue processing all the properties of the structural type
case
(
StructuralTypeShaperExpression { ValueBufferExpression: StructuralTypeProjectionExpression projection1 },
StructuralTypeShaperExpression { ValueBufferExpression: StructuralTypeProjectionExpression projection2 }
):
var resultComplexProjection = ProcessStructuralType(projection1, projection2);

var outerShaper = new RelationalStructuralTypeShaperExpression(
complexProperty.ComplexType,
resultComplexProjection,
resultComplexProjection.IsNullable);

complexPropertyCache[complexProperty] = outerShaper;
break;

// Set operation over type that contains a JSON-mapped complex type (non-collection)
case
(
StructuralTypeShaperExpression { ValueBufferExpression: JsonQueryExpression jsonQuery1 } shaper1,
StructuralTypeShaperExpression { ValueBufferExpression: JsonQueryExpression jsonQuery2 } shaper2
):
ProcessJson(jsonQuery1, jsonQuery2);
continue;

// Set operation over type that contains a JSON-mapped complex type (collection)
case
(
CollectionResultExpression { QueryExpression: JsonQueryExpression jsonQuery1 } collection1,
CollectionResultExpression { QueryExpression: JsonQueryExpression jsonQuery2 } collection2
):
ProcessJson(jsonQuery1, jsonQuery2);
continue;

default:
throw new UnreachableException();
}

var resultComplexProjection = ProcessStructuralType(
(StructuralTypeProjectionExpression)nonCollectionShaper1.ValueBufferExpression,
(StructuralTypeProjectionExpression)nonCollectionShaper2.ValueBufferExpression);
void ProcessJson(JsonQueryExpression jsonQuery1, JsonQueryExpression jsonQuery2)
{
Check.DebugAssert(jsonQuery1.StructuralType == jsonQuery2.StructuralType);
Check.DebugAssert(jsonQuery1.Type == jsonQuery2.Type);

// Convert the JsonQueryExpression to a JsonScalarExpression, which is our current representation for a complex
// JSON in the SQL tree (as opposed to in the shaper) - see #36392.
var jsonScalar1 = new JsonScalarExpression(
jsonQuery1.JsonColumn,
jsonQuery1.Path,
jsonQuery1.Type,
jsonQuery1.JsonColumn.TypeMapping,
jsonQuery1.IsNullable);
var jsonScalar2 = new JsonScalarExpression(
jsonQuery2.JsonColumn,
jsonQuery2.Path,
jsonQuery2.Type,
jsonQuery2.JsonColumn.TypeMapping,
jsonQuery2.IsNullable);

var alias = GenerateUniqueColumnAlias(complexProperty.Name.ToString());
var innerProjection = new ProjectionExpression(jsonScalar1, alias);
select1._projection.Add(innerProjection);
select2._projection.Add(new ProjectionExpression(jsonScalar2, alias));

var outerColumn = CreateColumnExpression(innerProjection, setOperationAlias);
if (jsonScalar1.IsNullable || jsonScalar2.IsNullable)
{
outerColumn = outerColumn.MakeNullable();
}

var outerJsonQuery = new JsonQueryExpression(
jsonQuery1.StructuralType,
outerColumn,
keyPropertyMap: null, // For owned entities only, here we're processing a complex type
jsonQuery1.Path,
jsonQuery1.Type,
collection: jsonQuery1.IsCollection,
jsonQuery1.IsNullable || jsonQuery2.IsNullable);

var resultComplexShaper = new RelationalStructuralTypeShaperExpression(
complexProperty.ComplexType,
resultComplexProjection,
resultComplexProjection.IsNullable);
var outerShaper = new CollectionResultExpression(outerJsonQuery, complexProperty, complexProperty.ComplexType.ClrType);

complexPropertyCache[complexProperty] = resultComplexShaper;
complexPropertyCache[complexProperty] = outerShaper;
}
}

Check.DebugAssert(
Expand Down Expand Up @@ -4167,10 +4236,11 @@ private static ColumnExpression CreateColumnExpression(ProjectionExpression subq
column: subqueryProjection.Expression is ColumnExpression { Column: IColumnBase column } ? column : null,
subqueryProjection.Type,
subqueryProjection.Expression.TypeMapping!,
subqueryProjection.Expression switch
nullable: subqueryProjection.Expression switch
{
ColumnExpression c => c.IsNullable,
SqlConstantExpression c => c.Value is null,
JsonScalarExpression j => j.IsNullable,
_ => true
});

Expand Down
24 changes: 17 additions & 7 deletions src/EFCore.Sqlite.Core/Query/Internal/SqliteQueryableMethodTranslatingExpressionVisitor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -375,13 +375,10 @@ protected override ShapedQueryExpression TransformJsonQueryToTable(JsonQueryExpr
var jsonColumn = selectExpression.CreateColumnExpression(
jsonEachExpression, JsonEachValueColumnName, typeof(string), _typeMappingSource.FindMapping(typeof(string))); // TODO: nullable?

var containerColumnName = structuralType.GetContainerColumnName();
Check.DebugAssert(containerColumnName is not null, "JsonQueryExpression to entity type without a container column name");

// First step: build a SelectExpression that will execute json_each and project all properties and navigations out, e.g.
// (SELECT value ->> 'a' AS a, value ->> 'b' AS b FROM json_each(c."JsonColumn", '$.Something.SomeCollection')

// We're only interested in properties which actually exist in the JSON, filter out uninteresting shadow keys
// We're only interested in properties which actually exist in the JSON, filter out uninteresting synthetic keys
foreach (var property in structuralType.GetPropertiesInHierarchy())
{
if (property.GetJsonPropertyName() is string jsonPropertyName)
Expand All @@ -393,14 +390,14 @@ protected override ShapedQueryExpression TransformJsonQueryToTable(JsonQueryExpr

propertyJsonScalarExpression[projectionMember] = new JsonScalarExpression(
Copy link
Member

@AndriySvyryd AndriySvyryd Jul 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

containerColumnName does not seem to be used outside of the Assert.

The comment // We're only interested in properties which actually exist in the JSON, filter out uninteresting shadow keys is incorrect as you aren't filtering out shadow properties. Also, invert the condition to reduce nesting

Copy link
Member Author

@roji roji Jul 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Re shadow keys/properties, the intent here was to mean that we want to jump over the synthetic key properties, which don't actually exist in the JSON document - that's what the if (property.GetJsonPropertyName() is string jsonPropertyName) does.

I'll change the comment to filter out uninteresting synthetic keys to make it clearer.

jsonColumn,
new[] { new PathSegment(property.GetJsonPropertyName()!) },
[new PathSegment(property.GetJsonPropertyName()!)],
property.ClrType.UnwrapNullableType(),
property.GetRelationalTypeMapping(),
property.IsNullable);
}
}

if (jsonQueryExpression.StructuralType is IEntityType entityType)
if (structuralType is IEntityType entityType)
{
foreach (var navigation in entityType.GetNavigationsInHierarchy()
.Where(
Expand All @@ -422,7 +419,20 @@ [new PathSegment(jsonNavigationName)],
}
}

// TODO: also add JsonScalarExpressions for complex properties, not just owned entities. #36296.
foreach (var complexProperty in structuralType.GetComplexProperties())
{
var jsonNavigationName = complexProperty.ComplexType.GetJsonPropertyName();
Check.DebugAssert(jsonNavigationName is not null, "Invalid complex property found on JSON-mapped structural type");

var projectionMember = new ProjectionMember().Append(new FakeMemberInfo(jsonNavigationName));

propertyJsonScalarExpression[projectionMember] = new JsonScalarExpression(
Copy link
Member

@AndriySvyryd AndriySvyryd Jul 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's strange that the type is called JsonScalarExpression when it also represents non-scalars

Copy link
Member Author

@roji roji Jul 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Tell me about it - this prompted me to file #36392, which contains the details. tl;dr JsonQueryExpression doesn't actually represent JSON_QUERY() in SQL, but rather represents a JSON being projected out of the database (so shaper-side expression only). Whe projections are applied (subquery pushdown, or when translation ends), any projected JsonQueryExpression gets converted to JsonScalarExpression.

... very questionable.

jsonColumn,
[new PathSegment(jsonNavigationName)],
typeof(string),
textTypeMapping,
jsonQueryExpression.IsNullable || complexProperty.IsNullable);
}

selectExpression.ReplaceProjection(propertyJsonScalarExpression);

Expand Down
18 changes: 11 additions & 7 deletions src/EFCore/Query/Internal/NavigationExpandingExpressionVisitor.ExpressionVisitors.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ protected override Expression VisitExtension(Expression expression)
protected override Expression VisitMember(MemberExpression memberExpression)
{
var innerExpression = Visit(memberExpression.Expression);
return TryExpandNavigation(innerExpression, MemberIdentity.Create(memberExpression.Member))
return TryExpandRelationship(innerExpression, MemberIdentity.Create(memberExpression.Member))
?? memberExpression.Update(innerExpression);
}

Expand All @@ -58,29 +58,30 @@ protected override Expression VisitMethodCall(MethodCallExpression methodCallExp
if (methodCallExpression.TryGetEFPropertyArguments(out var source, out var navigationName))
{
source = Visit(source);
return TryExpandNavigation(source, MemberIdentity.Create(navigationName))
return TryExpandRelationship(source, MemberIdentity.Create(navigationName))
?? methodCallExpression.Update(null, new[] { source, methodCallExpression.Arguments[1] });
}

if (methodCallExpression.TryGetIndexerArguments(Model, out source, out navigationName))
{
source = Visit(source);
return TryExpandNavigation(source, MemberIdentity.Create(navigationName))
return TryExpandRelationship(source, MemberIdentity.Create(navigationName))
?? methodCallExpression.Update(source, new[] { methodCallExpression.Arguments[0] });
}

return base.VisitMethodCall(methodCallExpression);
}

private Expression? TryExpandNavigation(Expression? root, MemberIdentity memberIdentity)
private Expression? TryExpandRelationship(Expression? root, MemberIdentity memberIdentity)
{
if (root == null)
{
return null;
}

var innerExpression = root.UnwrapTypeConversion(out var convertedType);
var entityReference = UnwrapEntityReference(innerExpression);
var structuralTypeReference = UnwrapStructuralTypeReference(innerExpression);
var entityReference = structuralTypeReference as EntityReference;
if (entityReference is not null)
{
var entityType = entityReference.EntityType;
Expand Down Expand Up @@ -117,8 +118,8 @@ protected override Expression VisitMethodCall(MethodCallExpression methodCallExp

var structuralType = entityReference is not null
? (ITypeBase)entityReference.EntityType
: innerExpression is ComplexPropertyReference complexReference
? complexReference.Property.ComplexType
: structuralTypeReference is ComplexTypeReference complexTypeReference
? complexTypeReference.ComplexType
: null;

if (structuralType is not null)
Expand Down Expand Up @@ -1101,6 +1102,9 @@ public IReadOnlyDictionary<NavigationTreeNode, NavigationTreeNode> ClonedNodesMa
case EntityReference entityReference:
return entityReference.Snapshot();

case ComplexTypeReference complexTypeReference:
return complexTypeReference;

case NavigationTreeExpression navigationTreeExpression:
if (!_clonedMap.TryGetValue(navigationTreeExpression, out var clonedNavigationTreeExpression))
{
Expand Down
23 changes: 21 additions & 2 deletions src/EFCore/Query/Internal/NavigationExpandingExpressionVisitor.Expressions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -485,7 +485,8 @@ void IPrintableExpression.Print(ExpressionPrinter expressionPrinter)
/// <summary>
/// Queryable properties are not expanded (similar to <see cref="OwnedNavigationReference" />.
/// </summary>
private sealed class ComplexPropertyReference(Expression parent, IComplexProperty property) : Expression, IPrintableExpression
private sealed class ComplexPropertyReference(Expression parent, IComplexProperty complexProperty)
: Expression, IPrintableExpression
{
protected override Expression VisitChildren(ExpressionVisitor visitor)
{
Expand All @@ -495,7 +496,8 @@ protected override Expression VisitChildren(ExpressionVisitor visitor)
}

public Expression Parent { get; private set; } = parent;
public new IComplexProperty Property { get; } = property;
public new IComplexProperty Property { get; } = complexProperty;
public ComplexTypeReference ComplexTypeReference { get; } = new(complexProperty.ComplexType);

public override Type Type
=> Property.ClrType;
Expand All @@ -516,6 +518,23 @@ void IPrintableExpression.Print(ExpressionPrinter expressionPrinter)
}
}

private sealed class ComplexTypeReference(IComplexType complexType) : Expression, IPrintableExpression
{
public IComplexType ComplexType { get; } = complexType;

public override ExpressionType NodeType
=> ExpressionType.Extension;

public override Type Type
=> ComplexType.ClrType;

protected override Expression VisitChildren(ExpressionVisitor visitor)
=> this;

void IPrintableExpression.Print(ExpressionPrinter expressionPrinter)
=> expressionPrinter.Append($"{nameof(ComplexTypeReference)}: {ComplexType.DisplayName()}");
}

/// <summary>
/// Queryable properties are not expanded (similar to <see cref="OwnedNavigationReference" />.
/// </summary>
Expand Down
23 changes: 19 additions & 4 deletions src/EFCore/Query/Internal/NavigationExpandingExpressionVisitor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,7 @@ protected override Expression VisitExtension(Expression extensionExpression)

case NavigationExpansionExpression:
case OwnedNavigationReference:
case ComplexPropertyReference:
return extensionExpression;

default:
Expand All @@ -268,6 +269,13 @@ protected override Expression VisitMember(MemberExpression memberExpression)
{
var innerExpression = Visit(memberExpression.Expression);

// Handler access of a complex collection property over a complex non-collection property
if (memberExpression.Expression is ComplexPropertyReference { Property: { IsCollection: false } complexProperty } complexPropertyReference
&& complexProperty.ComplexType.FindComplexProperty(memberExpression.Member) is IComplexProperty nestedComplexProperty)
{
return new ComplexPropertyReference(complexPropertyReference, nestedComplexProperty);
}

// Convert ICollection<T>.Count to Count<T>()
if (memberExpression.Expression != null
&& innerExpression != null
Expand Down Expand Up @@ -2079,9 +2087,9 @@ private Expression UnwrapCollectionMaterialization(Expression expression)
GetParameterName("o"));
}

case ComplexPropertyReference complexCollectionReference:
case ComplexPropertyReference { Property.IsCollection: true } complexCollectionReference:
{
var currentTree = new NavigationTreeExpression(Expression.Default(complexCollectionReference.Type.GetSequenceType()));
var currentTree = new NavigationTreeExpression(complexCollectionReference.ComplexTypeReference);

return new NavigationExpansionExpression(
Expression.Call(
Expand Down Expand Up @@ -2267,6 +2275,9 @@ private static Expression SnapshotExpression(Expression selector)
case EntityReference entityReference:
return entityReference.Snapshot();

case ComplexTypeReference complexTypeReference:
return complexTypeReference;

case NavigationTreeExpression navigationTreeExpression:
return SnapshotExpression(navigationTreeExpression.Value);

Expand Down Expand Up @@ -2294,13 +2305,17 @@ private static Expression SnapshotExpression(Expression selector)
}

private static EntityReference? UnwrapEntityReference(Expression? expression)
=> UnwrapStructuralTypeReference(expression) as EntityReference;

private static Expression? UnwrapStructuralTypeReference(Expression? expression)
=> expression switch
{
EntityReference entityReference => entityReference,
NavigationTreeExpression navigationTreeExpression => UnwrapEntityReference(navigationTreeExpression.Value),
ComplexTypeReference complexTypeReference => complexTypeReference,
NavigationTreeExpression navigationTreeExpression => UnwrapStructuralTypeReference(navigationTreeExpression.Value),
NavigationExpansionExpression navigationExpansionExpression
when navigationExpansionExpression.CardinalityReducingGenericMethodInfo is not null
=> UnwrapEntityReference(navigationExpansionExpression.PendingSelector),
=> UnwrapStructuralTypeReference(navigationExpansionExpression.PendingSelector),
OwnedNavigationReference ownedNavigationReference => ownedNavigationReference.EntityReference,

_ => null,
Expand Down
Loading