diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 7454720f872abe..11bc8cd3868824 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -4657,6 +4657,10 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl // DoPhase(this, PHASE_FIND_LOOPS, &Compiler::optFindLoopsPhase); + // Re-establish profile consistency, now that inlining and morph have run. + // + DoPhase(this, PHASE_REPAIR_PROFILE_POST_MORPH, &Compiler::fgRepairProfile); + // Scale block weights and mark run rarely blocks. // DoPhase(this, PHASE_SET_BLOCK_WEIGHTS, &Compiler::optSetBlockWeights); @@ -4962,9 +4966,9 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl // DoPhase(this, PHASE_OPTIMIZE_PRE_LAYOUT, &Compiler::optOptimizePreLayout); - // Run profile repair + // Ensure profile is consistent before starting backend phases // - DoPhase(this, PHASE_REPAIR_PROFILE, &Compiler::fgRepairProfile); + DoPhase(this, PHASE_REPAIR_PROFILE_PRE_LAYOUT, &Compiler::fgRepairProfile); } #ifdef DEBUG diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 76a25f0ab29f42..891b33a169e46d 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -6139,8 +6139,6 @@ class Compiler void fgCompactBlock(BasicBlock* block); - BasicBlock* fgConnectFallThrough(BasicBlock* bSrc, BasicBlock* bDst); - bool fgRenumberBlocks(); bool fgExpandRarelyRunBlocks(); diff --git a/src/coreclr/jit/compphases.h b/src/coreclr/jit/compphases.h index c016e270301745..21915bf4a13a2f 100644 --- a/src/coreclr/jit/compphases.h +++ b/src/coreclr/jit/compphases.h @@ -116,7 +116,8 @@ CompPhaseNameMacro(PHASE_INSERT_GC_POLLS, "Insert GC Polls", CompPhaseNameMacro(PHASE_CREATE_THROW_HELPERS, "Create throw helper blocks", false, -1, true) CompPhaseNameMacro(PHASE_DETERMINE_FIRST_COLD_BLOCK, "Determine first cold block", false, -1, true) CompPhaseNameMacro(PHASE_RATIONALIZE, "Rationalize IR", false, -1, false) -CompPhaseNameMacro(PHASE_REPAIR_PROFILE, "Repair profile", false, -1, false) +CompPhaseNameMacro(PHASE_REPAIR_PROFILE_POST_MORPH, "Repair profile post-morph", false, -1, false) +CompPhaseNameMacro(PHASE_REPAIR_PROFILE_PRE_LAYOUT, "Repair profile pre-layout", false, -1, false) CompPhaseNameMacro(PHASE_LCLVARLIVENESS, "Local var liveness", true, -1, false) CompPhaseNameMacro(PHASE_LCLVARLIVENESS_INIT, "Local var liveness init", false, PHASE_LCLVARLIVENESS, false) diff --git a/src/coreclr/jit/fgbasic.cpp b/src/coreclr/jit/fgbasic.cpp index a390b342d80bb6..9905526766a9cf 100644 --- a/src/coreclr/jit/fgbasic.cpp +++ b/src/coreclr/jit/fgbasic.cpp @@ -5248,68 +5248,6 @@ void Compiler::fgPrepareCallFinallyRetForRemoval(BasicBlock* block) block->SetKind(BBJ_ALWAYS); } -//------------------------------------------------------------------------ -// fgConnectFallThrough: fix flow from a block that previously had a fall through -// -// Arguments: -// bSrc - source of fall through -// bDst - target of fall through -// -// Returns: -// Newly inserted block after bSrc that jumps to bDst, -// or nullptr if bSrc already falls through to bDst -// -BasicBlock* Compiler::fgConnectFallThrough(BasicBlock* bSrc, BasicBlock* bDst) -{ - assert(bSrc != nullptr); - assert(fgPredsComputed); - BasicBlock* jmpBlk = nullptr; - - /* If bSrc falls through to a block that is not bDst, we will insert a jump to bDst */ - - if (bSrc->KindIs(BBJ_COND) && bSrc->FalseTargetIs(bDst) && !bSrc->NextIs(bDst)) - { - // Add a new block after bSrc which jumps to 'bDst' - jmpBlk = fgNewBBafter(BBJ_ALWAYS, bSrc, true); - FlowEdge* const oldEdge = bSrc->GetFalseEdge(); - - // Access the likelihood of oldEdge before - // it gets reset by SetTargetEdge below. - // - FlowEdge* const newEdge = fgAddRefPred(jmpBlk, bSrc, oldEdge); - fgReplacePred(oldEdge, jmpBlk); - jmpBlk->SetTargetEdge(oldEdge); - assert(jmpBlk->TargetIs(bDst)); - bSrc->SetFalseEdge(newEdge); - - // When adding a new jmpBlk we will set the bbWeight and bbFlags - // - if (fgHaveProfileWeights()) - { - jmpBlk->setBBProfileWeight(newEdge->getLikelyWeight()); - } - else - { - // We set the bbWeight to the smaller of bSrc->bbWeight or bDst->bbWeight - if (bSrc->bbWeight < bDst->bbWeight) - { - jmpBlk->bbWeight = bSrc->bbWeight; - jmpBlk->CopyFlags(bSrc, BBF_RUN_RARELY); - } - else - { - jmpBlk->bbWeight = bDst->bbWeight; - jmpBlk->CopyFlags(bDst, BBF_RUN_RARELY); - } - } - - JITDUMP("Added an unconditional jump to " FMT_BB " after block " FMT_BB "\n", jmpBlk->GetTarget()->bbNum, - bSrc->bbNum); - } - - return jmpBlk; -} - //------------------------------------------------------------------------ // fgRenumberBlocks: update block bbNums to reflect bbNext order // diff --git a/src/coreclr/jit/fgprofilesynthesis.cpp b/src/coreclr/jit/fgprofilesynthesis.cpp index 65b1499d731131..b045ef8a23e666 100644 --- a/src/coreclr/jit/fgprofilesynthesis.cpp +++ b/src/coreclr/jit/fgprofilesynthesis.cpp @@ -30,26 +30,6 @@ // void ProfileSynthesis::Run(ProfileSynthesisOption option) { - if (m_dfsTree == nullptr) - { - m_dfsTree = m_comp->fgComputeDfs(); - m_loops = FlowGraphNaturalLoops::Find(m_dfsTree); - m_improperLoopHeaders = m_loops->ImproperLoopHeaders(); - } - else - { - assert(m_loops != nullptr); - } - - if (m_loops->NumLoops() > 0) - { - m_cyclicProbabilities = new (m_comp, CMK_Pgo) weight_t[m_loops->NumLoops()]; - } - - // Profile synthesis can be run before or after morph, so tolerate (non-)canonical method entries - // - m_entryBlock = (m_comp->opts.IsOSR() && (m_comp->fgEntryBB != nullptr)) ? m_comp->fgEntryBB : m_comp->fgFirstBB; - // Retain or compute edge likelihood information // switch (option) diff --git a/src/coreclr/jit/fgprofilesynthesis.h b/src/coreclr/jit/fgprofilesynthesis.h index f27466835be542..b1425167c752c1 100644 --- a/src/coreclr/jit/fgprofilesynthesis.h +++ b/src/coreclr/jit/fgprofilesynthesis.h @@ -42,7 +42,29 @@ class ProfileSynthesis private: ProfileSynthesis(Compiler* compiler) : m_comp(compiler) + , m_dfsTree(compiler->m_dfsTree) + , m_loops(compiler->m_loops) + // Profile synthesis can be run before or after morph, so tolerate (non-)canonical method entries + , m_entryBlock((compiler->opts.IsOSR() && (compiler->fgEntryBB != nullptr)) ? compiler->fgEntryBB + : compiler->fgFirstBB) { + // If the Compiler object didn't give us flowgraph annotations to use, re-compute them + if (m_dfsTree == nullptr) + { + m_dfsTree = compiler->fgComputeDfs(); + m_loops = FlowGraphNaturalLoops::Find(m_dfsTree); + } + else + { + assert(m_loops != nullptr); + } + + m_improperLoopHeaders = m_loops->ImproperLoopHeaders(); + + if (m_loops->NumLoops() > 0) + { + m_cyclicProbabilities = new (compiler, CMK_Pgo) weight_t[m_loops->NumLoops()]; + } } static constexpr weight_t exceptionWeight = 0.00001;