Implement optional complex types

src/EFCore.Relational/Properties/RelationalStrings.resx

@@ -130,12 +130,6 @@
   <data name="CannotChangeWhenOpen" xml:space="preserve">
     <value>The instance of DbConnection is currently in use. The connection can only be changed when the existing connection is not being used.</value>
   </data>
-  <data name="CannotCompareComplexTypeToNull" xml:space="preserve">
-    <value>Comparing complex types to null is not supported.</value>
-  </data>
-  <data name="CannotProjectNullableComplexType" xml:space="preserve">
-    <value>You are attempting to project out complex type '{complexType}' via an optional navigation; that is currently not supported. Either project out the complex type in a non-optional context, or project the containing entity type along with the complex type.</value>
-  </data>
   <data name="CannotSetAliasOnJoin" xml:space="preserve">
     <value>Join expressions have no aliases; set the alias on the enclosed table expression.</value>
   </data>

src/EFCore.Relational/Query/Internal/RelationalStructuralTypeMaterializerSource.cs

@@ -1,9 +1,10 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+
 namespace Microsoft.EntityFrameworkCore.Query.Internal;
 
-#pragma warning disable EF1001 // EntityMaterializerSource is pubternal
+#pragma warning disable EF1001 // StructuralTypeMaterializerSource is pubternal
 
 /// <summary>
 ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
@@ -15,39 +16,9 @@ public class RelationalStructuralTypeMaterializerSource(StructuralTypeMaterializ
     : StructuralTypeMaterializerSource(dependencies)
 {
     /// <summary>
-    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
-    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
-    ///     any release. You should only use it directly in your code with extreme caution and knowing that
-    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
-    /// </summary>
-    protected override void AddInitializeExpression(
-        IPropertyBase property,
-        ParameterBindingInfo bindingInfo,
-        Expression instanceVariable,
-        MethodCallExpression valueBufferExpression,
-        List<Expression> blockExpressions)
-    {
-        // JSON complex properties are not handled in the initial materialization expression, since they're not
-        // simply e.g. DbDataReader.GetFieldValue<>() calls. So they're handled afterwards in the shaper, and need
-        // to be skipped here.
-        if (property is IComplexProperty { ComplexType: var complexType } && complexType.IsMappedToJson())
-        {
-            return;
-        }
-
-        base.AddInitializeExpression(property, bindingInfo, instanceVariable, valueBufferExpression, blockExpressions);
-    }
-
-    /// <summary>
-    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
-    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
-    ///     any release. You should only use it directly in your code with extreme caution and knowing that
-    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    ///     JSON complex properties are not handled in the initial materialization expression,
+    ///     since they're not simply e.g. DbDataReader.GetFieldValue calls.
+    ///     So they're handled afterwards in the shaper, and need to be skipped.
     /// </summary>
-    [EntityFrameworkInternal]
-    public static readonly MethodInfo MaterializeJsonComplexTypeMethod
-        = typeof(RelationalStructuralTypeMaterializerSource).GetTypeInfo().GetDeclaredMethod(nameof(MaterializeJsonComplexType))!;
-
-    private static T MaterializeJsonComplexType<T>(in ValueBuffer valueBuffer, IComplexProperty complexProperty)
-        => throw new UnreachableException();
+    protected override bool ReadComplexTypeDirectly(IComplexType complexType) => !complexType.IsMappedToJson();
 }

src/EFCore.Relational/Query/RelationalSqlTranslatingExpressionVisitor.StructuralEquality.cs

@@ -311,15 +311,6 @@ bool TryRewriteComplexTypeEquality(bool collection, [NotNullWhen(true)] out SqlE
 
             Check.DebugAssert(complexType != null, "We checked that at least one side is a complex type before calling this function");
 
-            // Comparison to null needs to be handled in a special way for table splitting, but for JSON mapping is handled via
-            // the regular JSON flow below.
-            if ((IsNullSqlConstantExpression(left) || IsNullSqlConstantExpression(right)) && !complexType.IsMappedToJson())
-            {
-                // TODO: when we support optional complex types with table splitting - or projecting required complex types via optional
-                // navigations - we'll be able to translate this, #31376
-                throw new InvalidOperationException(RelationalStrings.CannotCompareComplexTypeToNull);
-            }
-
             // If a complex type is the result of a subquery, then comparing its columns would mean duplicating the subquery, which would
             // be potentially very inefficient.
             // TODO: Enable this by extracting the subquery out to a common table expressions (WITH), #31237

src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs

@@ -492,12 +492,6 @@ public void ApplyProjection()
 
         void AddStructuralTypeProjection(StructuralTypeProjectionExpression projection)
         {
-            if (_projection.Count == 0
-                && projection is { StructuralType: IComplexType complexType, IsNullable: true })
-            {
-                throw new InvalidOperationException(RelationalStrings.CannotProjectNullableComplexType(complexType.DisplayName()));
-            }
-
             ProcessTypeProjection(projection);
 
             void ProcessTypeProjection(StructuralTypeProjectionExpression projection)
@@ -1367,11 +1361,6 @@ Expression CopyProjectionToOuter(SelectExpression innerSelectExpression, Express
 
         ConstantExpression AddStructuralTypeProjection(StructuralTypeProjectionExpression projection)
         {
-            if (projection is { StructuralType: IComplexType complexType, IsNullable: true })
-            {
-                throw new InvalidOperationException(RelationalStrings.CannotProjectNullableComplexType(complexType.DisplayName()));
-            }
-
             // JSON entity that had some query operations applied on it - it has been converted to a query root via OPENJSON/json_each
             // so it requires different materialization path than regular entity
             // e.g. we need to also add all the child navigations, JSON entity builds all the includes as part of it's own materializer

src/EFCore/Query/EntityMaterializerSourceParameters.cs

@@ -8,12 +8,14 @@ namespace Microsoft.EntityFrameworkCore.Query;
 /// </summary>
 /// <param name="StructuralType">The entity or complex type being materialized.</param>
 /// <param name="InstanceName">The name of the instance being materialized.</param>
+/// <param name="AllowNullable">Whether nullable result is allowed.</param>
 /// <param name="QueryTrackingBehavior">
 ///     The query tracking behavior, or <see langword="null" /> if this materialization is not from a query.
 /// </param>
 public readonly record struct StructuralTypeMaterializerSourceParameters(
     ITypeBase StructuralType,
     string InstanceName,
+    bool? AllowNullable,
     QueryTrackingBehavior? QueryTrackingBehavior);
 
 /// <summary>

src/EFCore/Query/Internal/StructuralTypeMaterializerSource.cs

@@ -95,28 +95,46 @@ public Expression CreateMaterializeExpression(
             properties.Remove(consumedProperty);
         }
 
-        var constructorExpression = constructorBinding.CreateConstructorExpression(bindingInfo);
-
-        if (_materializationInterceptor == null
-            // TODO: This currently applies the materialization interceptor only on the root structural type - any contained complex types
-            // don't get intercepted. #35883
-            || structuralType is not IEntityType)
+        var materializationExpression = HandleMaterializationInterception();
+
+        return
+            structuralType is IComplexType complexType && ReadComplexTypeDirectly(complexType)
+                && (IsNullable(complexType) || parameters.AllowNullable == true)
+                ? HandleNullableComplexTypeMaterialization(
+                    complexType,
+                    complexType.ClrType,
+                    materializationExpression,
+                    bindingInfo)
+                : materializationExpression;
+
+        Expression HandleMaterializationInterception()
         {
-            return properties.Count == 0 && blockExpressions.Count == 0
-                ? constructorExpression
-                : CreateMaterializeExpression(blockExpressions, instanceVariable, constructorExpression, properties, bindingInfo);
+            var constructorExpression = constructorBinding.CreateConstructorExpression(bindingInfo);
+
+            return _materializationInterceptor == null
+                // TODO: This currently applies the materialization interceptor only on the root structural type - any contained complex types
+                // don't get intercepted. #35883
+                || structuralType is not IEntityType
+                ? properties.Count == 0 && blockExpressions.Count == 0
+                    ? constructorExpression
+                    : CreateMaterializeExpression(blockExpressions, instanceVariable, constructorExpression, properties, bindingInfo)
+                : CreateInterceptionMaterializeExpression(
+                    structuralType,
+                    properties,
+                    _materializationInterceptor,
+                    bindingInfo,
+                    constructorExpression,
+                    instanceVariable,
+                    blockExpressions);
         }
-
-        return CreateInterceptionMaterializeExpression(
-            structuralType,
-            properties,
-            _materializationInterceptor,
-            bindingInfo,
-            constructorExpression,
-            instanceVariable,
-            blockExpressions);
     }
 
+    /// <summary>
+    ///     Should complex type be read directly using e.g. DbDataReader.GetFieldValue
+    ///     or is it going to be handled separately (i.e. relational JSON).
+    /// </summary>
+    protected virtual bool ReadComplexTypeDirectly(IComplexType complexType) => true;
+
     /// <summary>
     ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
     ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
@@ -130,11 +148,17 @@ protected virtual void AddInitializeExpression(
         MethodCallExpression valueBufferExpression,
         List<Expression> blockExpressions)
     {
+        if (property is IComplexProperty cp && !ReadComplexTypeDirectly(cp.ComplexType))
+        {
+            return;
+        }
+
         var memberInfo = property.GetMemberInfo(forMaterialization: true, forSet: true);
 
         var valueExpression = property switch
         {
-            IProperty p => valueBufferExpression.CreateValueBufferReadValueExpression(memberInfo.GetMemberType(), p.GetIndex(), p),
+            IProperty p
+                => valueBufferExpression.CreateValueBufferReadValueExpression(memberInfo.GetMemberType(), p.GetIndex(), p),
 
             IServiceProperty serviceProperty
                 => serviceProperty.ParameterBinding.BindToParameter(bindingInfo),
@@ -143,9 +167,7 @@ IServiceProperty serviceProperty
                 => Default(complexProperty.ClrType), // Initialize collections to null, they'll be populated separately
 
             IComplexProperty complexProperty
-                => CreateMaterializeExpression(
-                    new StructuralTypeMaterializerSourceParameters(complexProperty.ComplexType, "complexType", QueryTrackingBehavior: null),
-                    bindingInfo.MaterializationContextExpression),
+                => CreateComplexTypeMaterializeExpression(complexProperty, bindingInfo),
 
             _ => throw new UnreachableException()
         };
@@ -193,6 +215,21 @@ static Expression CreateMemberAssignment(Expression parameter, MemberInfo member
                     value)
                 : MakeMemberAccess(parameter, memberInfo).Assign(value);
         }
+
+        Expression CreateComplexTypeMaterializeExpression(IComplexProperty complexProperty, ParameterBindingInfo bindingInfo)
+        {
+            var materializeExpression = CreateMaterializeExpression(
+                new StructuralTypeMaterializerSourceParameters(complexProperty.ComplexType, "complexType", null, QueryTrackingBehavior: null),
+                bindingInfo.MaterializationContextExpression);
+
+            return IsNullable(complexProperty)
+                ? HandleNullableComplexTypeMaterialization(
+                    complexProperty.ComplexType,
+                    complexProperty.ClrType,
+                    materializeExpression,
+                    bindingInfo)
+                : materializeExpression;
+        }
     }
 
     private void AddInitializeExpressions(
@@ -493,15 +530,14 @@ BlockExpression CreateInitializeExpression()
     ///     any release. You should only use it directly in your code with extreme caution and knowing that
     ///     doing so can result in application failures when updating to a new Entity Framework Core release.
     /// </summary>
-    public virtual Func<MaterializationContext, object> GetMaterializer(
-        IEntityType entityType)
+    public virtual Func<MaterializationContext, object> GetMaterializer(IEntityType entityType)
     {
         var materializationContextParameter
             = Parameter(typeof(MaterializationContext), "materializationContext");
 
         return Lambda<Func<MaterializationContext, object>>(
                 ((IStructuralTypeMaterializerSource)this).CreateMaterializeExpression(
-                    new StructuralTypeMaterializerSourceParameters(entityType, "instance", null), materializationContextParameter),
+                    new StructuralTypeMaterializerSourceParameters(entityType, "instance", null, null), materializationContextParameter),
                 materializationContextParameter)
             .Compile();
     }
@@ -518,7 +554,7 @@ public virtual Func<MaterializationContext, object> GetMaterializer(IComplexType
 
         return Lambda<Func<MaterializationContext, object>>(
                 ((IStructuralTypeMaterializerSource)this).CreateMaterializeExpression(
-                    new StructuralTypeMaterializerSourceParameters(complexType, "instance", null), materializationContextParameter),
+                    new StructuralTypeMaterializerSourceParameters(complexType, "instance", null, null), materializationContextParameter),
                 materializationContextParameter)
             .Compile();
     }
@@ -572,7 +608,7 @@ public virtual Func<MaterializationContext, object> GetEmptyMaterializer(
 
         var materializationContextExpression = Parameter(typeof(MaterializationContext), "mc");
         var bindingInfo = new ParameterBindingInfo(
-            new StructuralTypeMaterializerSourceParameters(entityType, "instance", null), materializationContextExpression);
+            new StructuralTypeMaterializerSourceParameters(entityType, "instance", null, null), materializationContextExpression);
 
         var blockExpressions = new List<Expression>();
         var instanceVariable = Variable(binding.RuntimeType, "instance");
@@ -644,4 +680,50 @@ private static void CreateServiceInstances(
             }
         }
     }
+
+    private Expression HandleNullableComplexTypeMaterialization(IComplexType complexType, Type clrType, Expression materializeExpression, ParameterBindingInfo bindingInfo)
+    {
+        var valueBufferExpression = Call(
+            bindingInfo.MaterializationContextExpression,
+            MaterializationContext.GetValueBufferMethod);
+
+        // Get all scalar properties of the complex type (including nested ones).
+        var allScalarProperties = complexType.GetFlattenedProperties().ToList();
+
+        if (allScalarProperties is [])
+        {
+            // If no scalar properties, just create the instance.
+            return CreateMaterializeExpression(
+                new StructuralTypeMaterializerSourceParameters(complexType, "complexType", null, QueryTrackingBehavior: null),
+                bindingInfo.MaterializationContextExpression);
+        }
+
+        var requiredProperty = allScalarProperties.Where(p => !p.IsNullable).FirstOrDefault();
+        var nullCheck = requiredProperty is not null
+            // If there's a required property, it's enough to check just that one for null.
+            ? Equal(
+                valueBufferExpression.CreateValueBufferReadValueExpression(typeof(object), requiredProperty.GetIndex(), requiredProperty),
+                Constant(null, typeof(object)))
+            // Create null checks for all scalar properties.
+            : allScalarProperties
+                .Select(p =>
+                    Equal(
+                        valueBufferExpression.CreateValueBufferReadValueExpression(typeof(object), p.GetIndex(), p),
+                        Constant(null, typeof(object))))
+                .Aggregate(AndAlso);
+
+        // If property/properties are null, return default (to handle structs); otherwise materialize the complex type.
+        return Condition(
+            nullCheck,
+            Default(clrType),
+            materializeExpression);
+    }
+
+    private static bool IsNullable(IComplexType complexType)
+        => IsNullable(complexType.ComplexProperty);
+
+    private static bool IsNullable(IComplexProperty complexProperty)
+        => complexProperty.IsNullable
+            || (complexProperty.DeclaringType is IComplexType complexType
+                && IsNullable(complexType.ComplexProperty));
 }