Inline nullable allocators

src/coreclr/jit/compiler.cpp

@@ -4349,6 +4349,10 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl
         return;
     }
 
+    DoPhase(this, PHASE_EARLY_QMARK_EXPANSION, [this]() {
+        return fgExpandQmarkNodes(/*early*/ true);
+    });
+
     // If instrumenting, add block and class probes.
     //
     if (compileFlags->IsSet(JitFlags::JIT_FLAG_BBINSTR))
@@ -4546,7 +4550,7 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl
         // Decide the kind of code we want to generate
         fgSetOptions();
 
-        fgExpandQmarkNodes();
+        fgExpandQmarkNodes(/*early*/ false);
 
 #ifdef DEBUG
         compCurBB = nullptr;

src/coreclr/jit/compiler.h

@@ -5565,7 +5565,7 @@ class Compiler
 
     GenTreeQmark* fgGetTopLevelQmark(GenTree* expr, GenTree** ppDst = nullptr);
     bool fgExpandQmarkStmt(BasicBlock* block, Statement* stmt);
-    void fgExpandQmarkNodes();
+    PhaseStatus fgExpandQmarkNodes(bool early);
 
     bool fgSimpleLowerCastOfSmpOp(LIR::Range& range, GenTreeCast* cast);
     bool fgSimpleLowerBswap16(LIR::Range& range, GenTree* op);
@@ -7426,6 +7426,7 @@ class Compiler
 #define OMF_HAS_EXPANDABLE_CAST                0x00080000 // Method contains casts eligible for late expansion
 #define OMF_HAS_STACK_ARRAY                    0x00100000 // Method contains stack allocated arrays
 #define OMF_HAS_BOUNDS_CHECKS                  0x00200000 // Method contains bounds checks
+#define OMF_HAS_EARLY_QMARKS                   0x00400000 // Method contains early expandable QMARKs
 
     // clang-format on
 

src/coreclr/jit/compphases.h

@@ -63,6 +63,7 @@ CompPhaseNameMacro(PHASE_GS_COOKIE,                  "GS Cookie",
 CompPhaseNameMacro(PHASE_COMPUTE_BLOCK_WEIGHTS,      "Compute block weights",          false, -1, false)
 CompPhaseNameMacro(PHASE_CREATE_FUNCLETS,            "Create EH funclets",             false, -1, false)
 CompPhaseNameMacro(PHASE_HEAD_TAIL_MERGE,            "Head and tail merge",            false, -1, false)
+CompPhaseNameMacro(PHASE_EARLY_QMARK_EXPANSION,      "Early QMARK expansion",          false, -1, false)
 CompPhaseNameMacro(PHASE_MERGE_THROWS,               "Merge throw blocks",             false, -1, false)
 CompPhaseNameMacro(PHASE_INVERT_LOOPS,               "Invert loops",                   false, -1, false)
 CompPhaseNameMacro(PHASE_HEAD_TAIL_MERGE2,           "Post-morph head and tail merge", false, -1, false)

src/coreclr/jit/importer.cpp

@@ -3414,6 +3414,76 @@ void Compiler::impImportAndPushBox(CORINFO_RESOLVED_TOKEN* pResolvedToken)
     // Look at what helper we should use.
     CorInfoHelpFunc boxHelper = info.compCompHnd->getBoxHelper(pResolvedToken->hClass);
 
+    // Expand nullable boxes inline in optimized code in hot paths, it's slightly different from
+    // other expansions since we have to use QMARK to return null for nullables with no value.
+    //
+    //  obj = nullable._hasValue == 0 ? null : (boxed underlying value)
+    //
+    if ((boxHelper == CORINFO_HELP_BOX_NULLABLE) && opts.OptimizationEnabled() && !compCurBB->isRunRarely() &&
+        !eeIsSharedInst(pResolvedToken->hClass))
+    {
+        CORINFO_CLASS_HANDLE typeToBox = info.compCompHnd->getTypeForBox(pResolvedToken->hClass);
+
+        // We need to compose a new CORINFO_RESOLVED_TOKEN for the underlying (typeToBox) type
+        CORINFO_RESOLVED_TOKEN tk = *pResolvedToken;
+        tk.hClass                 = typeToBox;
+        tk.tokenType              = CORINFO_TOKENKIND_Casting;
+        GenTree* obj              = gtNewAllocObjNode(&tk, info.compMethodHnd, false);
+
+        if (obj != nullptr)
+        {
+            // First, decompose the Nullable<> into _hasValue and _value fields.
+            //
+            GenTreeFlags indirFlags             = GTF_EMPTY;
+            exprToBox                           = impGetNodeAddr(exprToBox, CHECK_SPILL_ALL, &indirFlags);
+            CORINFO_FIELD_HANDLE valueFldHnd    = info.compCompHnd->getFieldInClass(pResolvedToken->hClass, 1);
+            CORINFO_CLASS_HANDLE valueStructCls = NO_CLASS_HANDLE;
+            static_assert(OFFSETOF__CORINFO_NullableOfT__hasValue == 0);
+            unsigned     cnsValueOffset = info.compCompHnd->getFieldOffset(valueFldHnd);
+            ClassLayout* layout         = nullptr;
+            CorInfoType  corFldType     = info.compCompHnd->getFieldType(valueFldHnd, &valueStructCls);
+            var_types    valueType      = TypeHandleToVarType(corFldType, valueStructCls, &layout);
+            GenTree*     valueOffset    = gtNewIconNode(cnsValueOffset, TYP_I_IMPL);
+            GenTree*     valueAddr      = gtNewOperNode(GT_ADD, TYP_BYREF, gtCloneExpr(exprToBox), valueOffset);
+            GenTree*     value          = gtNewLoadValueNode(valueType, layout, valueAddr);
+            GenTree*     hasValue       = gtNewLoadValueNode(TYP_UBYTE, nullptr, gtCloneExpr(exprToBox));
+
+            // Now we need to copy value into the allocated box
+            //
+            unsigned   objLclNum  = lvaGrabTemp(true DEBUGARG("obj nullable box"));
+            GenTree*   storeAlloc = gtNewTempStore(objLclNum, obj);
+            GenTree*   objLcl     = gtNewLclvNode(objLclNum, genActualType(obj));
+            GenTree*   pOffset    = gtNewIconNode(TARGET_POINTER_SIZE, TYP_I_IMPL);
+            GenTreeOp* dataPtr    = gtNewOperNode(GT_ADD, TYP_BYREF, gtCloneExpr(objLcl), pOffset);
+            GenTree*   storeData =
+                gtNewStoreValueNode(valueType, layout, dataPtr, gtCloneExpr(value), GTF_IND_NONFAULTING);
+
+            // Wrap it all in two commas, it will look like:
+            //   lcl = allocobj
+            //   *(lcl + sizeof(void*)) = value
+            //   lcl
+            //
+            GenTreeOp* copyData  = gtNewOperNode(GT_COMMA, TYP_REF, storeData, gtCloneExpr(objLcl));
+            GenTreeOp* allocRoot = gtNewOperNode(GT_COMMA, TYP_REF, storeAlloc, copyData);
+
+            // QMARK expansion will propagate block flags properly.
+            compCurBB->SetFlags(BBF_HAS_NEWOBJ);
+            optMethodFlags |= OMF_HAS_NEWOBJ | OMF_HAS_EARLY_QMARKS;
+
+            GenTree*      cond  = gtNewOperNode(GT_EQ, TYP_INT, hasValue, gtNewIconNode(0));
+            GenTreeColon* colon = gtNewColonNode(TYP_REF, gtNewNull(), allocRoot);
+            GenTree*      qmark = gtNewQmarkNode(TYP_REF, cond, colon);
+
+            // QMARK has to be a top-level statement
+            const unsigned result = lvaGrabTemp(true DEBUGARG("spilling qmarkNullableBox"));
+            impStoreToTemp(result, qmark, CHECK_SPILL_ALL);
+            lvaSetClass(result, typeToBox, true);
+            lvaSetClass(objLclNum, typeToBox, true);
+            impPushOnStack(gtNewLclvNode(result, TYP_REF), typeInfo(typeToBox));
+            return;
+        }
+    }
+
     // Determine what expansion to prefer.
     //
     // In size/time/debuggable constrained modes, the helper call

src/coreclr/jit/morph.cpp

@@ -14642,6 +14642,14 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt)
         }
         else
         {
+            // There is no good reason to keep root COMMAs when we can split them into separate statements.
+            while (trueExpr->OperIs(GT_COMMA))
+            {
+                Statement* trueStmt = fgNewStmtFromTree(trueExpr->gtGetOp1(), stmt->GetDebugInfo());
+                trueExpr            = trueExpr->gtGetOp2();
+                fgInsertStmtAtEnd(thenBlock, trueStmt);
+            }
+
             if (dst != nullptr)
             {
                 trueExpr = dst->OperIs(GT_STORE_LCL_FLD) ? gtNewStoreLclFldNode(dstLclNum, dst->TypeGet(),
@@ -14665,6 +14673,14 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt)
         }
         else
         {
+            // There is no good reason to keep root COMMAs when we can split them into separate statements.
+            while (falseExpr->OperIs(GT_COMMA))
+            {
+                Statement* falseStmt = fgNewStmtFromTree(falseExpr->gtGetOp1(), stmt->GetDebugInfo());
+                falseExpr            = falseExpr->gtGetOp2();
+                fgInsertStmtAtEnd(elseBlock, falseStmt);
+            }
+
             if (dst != nullptr)
             {
                 falseExpr =
@@ -14688,14 +14704,23 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt)
     return introducedThrow;
 }
 
-/*****************************************************************************
- *
- *  Expand GT_QMARK nodes from the flow graph into basic blocks.
- *
- */
-
-void Compiler::fgExpandQmarkNodes()
+//------------------------------------------------------------------------
+// fgExpandQmarkNodes: expand all QMARK nodes into control flow.
+//
+// Arguments:
+//    early - whether this is the early expansion phase. Late expansion
+//            happens too late for some optimizations, e.g. object stack-alloc.
+//
+// Returns:
+//    Suitable phase status.
+//
+PhaseStatus Compiler::fgExpandQmarkNodes(bool early)
 {
+    if (early && ((optMethodFlags & OMF_HAS_EARLY_QMARKS) == 0))
+    {
+        return PhaseStatus::MODIFIED_NOTHING;
+    }
+
     bool introducedThrows = false;
 
     if (compQmarkUsed)
@@ -14715,6 +14740,7 @@ void Compiler::fgExpandQmarkNodes()
         fgPostExpandQmarkChecks();
 #endif
     }
+
     compQmarkRationalized = true;
 
     // TODO: if qmark expansion created throw blocks, try and merge them
@@ -14723,6 +14749,8 @@ void Compiler::fgExpandQmarkNodes()
     {
         JITDUMP("Qmark expansion created new throw blocks\n");
     }
+
+    return PhaseStatus::MODIFIED_EVERYTHING;
 }
 
 //------------------------------------------------------------------------