JIT: Add post-morph profile repair phase

src/coreclr/jit/compiler.cpp

@@ -4657,6 +4657,10 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl
         //
         DoPhase(this, PHASE_FIND_LOOPS, &Compiler::optFindLoopsPhase);
 
+        // Re-establish profile consistency, now that inlining and morph have run.
+        //
+        DoPhase(this, PHASE_REPAIR_PROFILE_POST_MORPH, &Compiler::fgRepairProfile);
+
         // Scale block weights and mark run rarely blocks.
         //
         DoPhase(this, PHASE_SET_BLOCK_WEIGHTS, &Compiler::optSetBlockWeights);
@@ -4962,9 +4966,9 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl
         //
         DoPhase(this, PHASE_OPTIMIZE_PRE_LAYOUT, &Compiler::optOptimizePreLayout);
 
-        // Run profile repair
+        // Ensure profile is consistent before starting backend phases
         //
-        DoPhase(this, PHASE_REPAIR_PROFILE, &Compiler::fgRepairProfile);
+        DoPhase(this, PHASE_REPAIR_PROFILE_PRE_LAYOUT, &Compiler::fgRepairProfile);
     }
 
 #ifdef DEBUG

src/coreclr/jit/compiler.h

@@ -6139,8 +6139,6 @@ class Compiler
 
     void fgCompactBlock(BasicBlock* block);
 
-    BasicBlock* fgConnectFallThrough(BasicBlock* bSrc, BasicBlock* bDst);
-
     bool fgRenumberBlocks();
 
     bool fgExpandRarelyRunBlocks();

src/coreclr/jit/compphases.h

@@ -116,7 +116,8 @@ CompPhaseNameMacro(PHASE_INSERT_GC_POLLS,            "Insert GC Polls",
 CompPhaseNameMacro(PHASE_CREATE_THROW_HELPERS,       "Create throw helper blocks",     false, -1, true)
 CompPhaseNameMacro(PHASE_DETERMINE_FIRST_COLD_BLOCK, "Determine first cold block",     false, -1, true)
 CompPhaseNameMacro(PHASE_RATIONALIZE,                "Rationalize IR",                 false, -1, false)
-CompPhaseNameMacro(PHASE_REPAIR_PROFILE,             "Repair profile",                 false, -1, false)
+CompPhaseNameMacro(PHASE_REPAIR_PROFILE_POST_MORPH,  "Repair profile post-morph",      false, -1, false)
+CompPhaseNameMacro(PHASE_REPAIR_PROFILE_PRE_LAYOUT,  "Repair profile pre-layout",      false, -1, false)
 
 CompPhaseNameMacro(PHASE_LCLVARLIVENESS,             "Local var liveness",             true,  -1, false)
 CompPhaseNameMacro(PHASE_LCLVARLIVENESS_INIT,        "Local var liveness init",        false, PHASE_LCLVARLIVENESS, false)

src/coreclr/jit/fgbasic.cpp

@@ -5248,68 +5248,6 @@ void Compiler::fgPrepareCallFinallyRetForRemoval(BasicBlock* block)
     block->SetKind(BBJ_ALWAYS);
 }
 
-//------------------------------------------------------------------------
-// fgConnectFallThrough: fix flow from a block that previously had a fall through
-//
-// Arguments:
-//   bSrc - source of fall through
-//   bDst - target of fall through
-//
-// Returns:
-//   Newly inserted block after bSrc that jumps to bDst,
-//   or nullptr if bSrc already falls through to bDst
-//
-BasicBlock* Compiler::fgConnectFallThrough(BasicBlock* bSrc, BasicBlock* bDst)
-{
-    assert(bSrc != nullptr);
-    assert(fgPredsComputed);
-    BasicBlock* jmpBlk = nullptr;
-
-    /* If bSrc falls through to a block that is not bDst, we will insert a jump to bDst */
-
-    if (bSrc->KindIs(BBJ_COND) && bSrc->FalseTargetIs(bDst) && !bSrc->NextIs(bDst))
-    {
-        // Add a new block after bSrc which jumps to 'bDst'
-        jmpBlk                  = fgNewBBafter(BBJ_ALWAYS, bSrc, true);
-        FlowEdge* const oldEdge = bSrc->GetFalseEdge();
-
-        // Access the likelihood of oldEdge before
-        // it gets reset by SetTargetEdge below.
-        //
-        FlowEdge* const newEdge = fgAddRefPred(jmpBlk, bSrc, oldEdge);
-        fgReplacePred(oldEdge, jmpBlk);
-        jmpBlk->SetTargetEdge(oldEdge);
-        assert(jmpBlk->TargetIs(bDst));
-        bSrc->SetFalseEdge(newEdge);
-
-        // When adding a new jmpBlk we will set the bbWeight and bbFlags
-        //
-        if (fgHaveProfileWeights())
-        {
-            jmpBlk->setBBProfileWeight(newEdge->getLikelyWeight());
-        }
-        else
-        {
-            // We set the bbWeight to the smaller of bSrc->bbWeight or bDst->bbWeight
-            if (bSrc->bbWeight < bDst->bbWeight)
-            {
-                jmpBlk->bbWeight = bSrc->bbWeight;
-                jmpBlk->CopyFlags(bSrc, BBF_RUN_RARELY);
-            }
-            else
-            {
-                jmpBlk->bbWeight = bDst->bbWeight;
-                jmpBlk->CopyFlags(bDst, BBF_RUN_RARELY);
-            }
-        }
-
-        JITDUMP("Added an unconditional jump to " FMT_BB " after block " FMT_BB "\n", jmpBlk->GetTarget()->bbNum,
-                bSrc->bbNum);
-    }
-
-    return jmpBlk;
-}
-
 //------------------------------------------------------------------------
 // fgRenumberBlocks: update block bbNums to reflect bbNext order
 //

src/coreclr/jit/fgprofilesynthesis.cpp

@@ -30,26 +30,6 @@
 //
 void ProfileSynthesis::Run(ProfileSynthesisOption option)
 {
-    if (m_dfsTree == nullptr)
-    {
-        m_dfsTree             = m_comp->fgComputeDfs();
-        m_loops               = FlowGraphNaturalLoops::Find(m_dfsTree);
-        m_improperLoopHeaders = m_loops->ImproperLoopHeaders();
-    }
-    else
-    {
-        assert(m_loops != nullptr);
-    }
-
-    if (m_loops->NumLoops() > 0)
-    {
-        m_cyclicProbabilities = new (m_comp, CMK_Pgo) weight_t[m_loops->NumLoops()];
-    }
-
-    // Profile synthesis can be run before or after morph, so tolerate (non-)canonical method entries
-    //
-    m_entryBlock = (m_comp->opts.IsOSR() && (m_comp->fgEntryBB != nullptr)) ? m_comp->fgEntryBB : m_comp->fgFirstBB;
-
     // Retain or compute edge likelihood information
     //
     switch (option)

src/coreclr/jit/fgprofilesynthesis.h

@@ -42,7 +42,29 @@ class ProfileSynthesis
 private:
     ProfileSynthesis(Compiler* compiler)
         : m_comp(compiler)
+        , m_dfsTree(compiler->m_dfsTree)
+        , m_loops(compiler->m_loops)
+        // Profile synthesis can be run before or after morph, so tolerate (non-)canonical method entries
+        , m_entryBlock((compiler->opts.IsOSR() && (compiler->fgEntryBB != nullptr)) ? compiler->fgEntryBB
+                                                                                    : compiler->fgFirstBB)
     {
+        // If the Compiler object didn't give us flowgraph annotations to use, re-compute them
+        if (m_dfsTree == nullptr)
+        {
+            m_dfsTree = compiler->fgComputeDfs();
+            m_loops   = FlowGraphNaturalLoops::Find(m_dfsTree);
+        }
+        else
+        {
+            assert(m_loops != nullptr);
+        }
+
+        m_improperLoopHeaders = m_loops->ImproperLoopHeaders();
+
+        if (m_loops->NumLoops() > 0)
+        {
+            m_cyclicProbabilities = new (compiler, CMK_Pgo) weight_t[m_loops->NumLoops()];
+        }
     }
 
     static constexpr weight_t exceptionWeight       = 0.00001;