Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

JIT: Use likelihood-based edge weights #99736

Merged
merged 19 commits into from
Apr 16, 2024
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 0 additions & 24 deletions src/coreclr/jit/block.h
Original file line number Diff line number Diff line change
Expand Up @@ -582,10 +582,6 @@ struct FlowEdge
// The destination of the control flow
BasicBlock* m_destBlock;

// Edge weights
weight_t m_edgeWeightMin;
weight_t m_edgeWeightMax;

// Likelihood that m_sourceBlock transfers control along this edge.
// Values in range [0..1]
weight_t m_likelihood;
Expand All @@ -601,8 +597,6 @@ struct FlowEdge
: m_nextPredEdge(rest)
, m_sourceBlock(sourceBlock)
, m_destBlock(destBlock)
, m_edgeWeightMin(0)
, m_edgeWeightMax(0)
, m_likelihood(0)
, m_dupCount(0)
, m_likelihoodSet(false)
Expand Down Expand Up @@ -648,24 +642,6 @@ struct FlowEdge
m_destBlock = newBlock;
}

weight_t edgeWeightMin() const
{
return m_edgeWeightMin;
}

weight_t edgeWeightMax() const
{
return m_edgeWeightMax;
}

// These two methods are used to set new values for edge weights.
// They return false if the newWeight is not between the current [min..max]
// when slop is non-zero we allow for the case where our weights might be off by 'slop'
//
bool setEdgeWeightMinChecked(weight_t newWeight, BasicBlock* bDst, weight_t slop, bool* wbUsedSlop);
bool setEdgeWeightMaxChecked(weight_t newWeight, BasicBlock* bDst, weight_t slop, bool* wbUsedSlop);
void setEdgeWeights(weight_t newMinWeight, weight_t newMaxWeight, BasicBlock* bDst);

weight_t getLikelihood() const
{
return m_likelihood;
Expand Down
3 changes: 1 addition & 2 deletions src/coreclr/jit/codegencommon.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1859,8 +1859,7 @@ void CodeGen::genGenerateMachineCode()

if (compiler->fgHaveProfileWeights())
{
printf("; with %s: edge weights are %s, and fgCalledCount is " FMT_WT "\n",
compiler->compGetPgoSourceName(), compiler->fgHaveValidEdgeWeights ? "valid" : "invalid",
printf("; with %s: fgCalledCount is " FMT_WT "\n", compiler->compGetPgoSourceName(),
compiler->fgCalledCount);
}

Expand Down
4 changes: 0 additions & 4 deletions src/coreclr/jit/compiler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5024,10 +5024,6 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl
// update the flowgraph if we modified it during the optimization phase
//
DoPhase(this, PHASE_OPT_UPDATE_FLOW_GRAPH, &Compiler::fgUpdateFlowGraphPhase);

// Recompute the edge weight if we have modified the flow graph
//
DoPhase(this, PHASE_COMPUTE_EDGE_WEIGHTS2, &Compiler::fgComputeEdgeWeights);
}

// Iterate if requested, resetting annotations first.
Expand Down
25 changes: 9 additions & 16 deletions src/coreclr/jit/compiler.h
Original file line number Diff line number Diff line change
Expand Up @@ -1582,12 +1582,11 @@ enum API_ICorJitInfo_Names
enum class ProfileChecks : unsigned int
{
CHECK_NONE = 0,
CHECK_CLASSIC = 1 << 0, // check "classic" jit weights
CHECK_HASLIKELIHOOD = 1 << 1, // check all FlowEdges for hasLikelihood
CHECK_LIKELIHOODSUM = 1 << 2, // check block succesor likelihoods sum to 1
CHECK_LIKELY = 1 << 3, // fully check likelihood based weights
RAISE_ASSERT = 1 << 4, // assert on check failure
CHECK_ALL_BLOCKS = 1 << 5, // check blocks even if bbHasProfileWeight is false
CHECK_HASLIKELIHOOD = 1 << 0, // check all FlowEdges for hasLikelihood
CHECK_LIKELIHOODSUM = 1 << 1, // check block succesor likelihoods sum to 1
CHECK_LIKELY = 1 << 2, // fully check likelihood based weights
RAISE_ASSERT = 1 << 3, // assert on check failure
CHECK_ALL_BLOCKS = 1 << 4, // check blocks even if bbHasProfileWeight is false
};

inline constexpr ProfileChecks operator ~(ProfileChecks a)
Expand Down Expand Up @@ -5175,14 +5174,10 @@ class Compiler
// - Rationalization links all nodes into linear form which is kept until
// the end of compilation. The first and last nodes are stored in the block.
NodeThreading fgNodeThreading;
bool fgCanRelocateEHRegions; // true if we are allowed to relocate the EH regions
bool fgEdgeWeightsComputed; // true after we have called fgComputeEdgeWeights
bool fgHaveValidEdgeWeights; // true if we were successful in computing all of the edge weights
bool fgSlopUsedInEdgeWeights; // true if their was some slop used when computing the edge weights
bool fgRangeUsedInEdgeWeights; // true if some of the edgeWeight are expressed in Min..Max form
weight_t fgCalledCount; // count of the number of times this method was called
// This is derived from the profile data
// or is BB_UNITY_WEIGHT when we don't have profile data
bool fgCanRelocateEHRegions; // true if we are allowed to relocate the EH regions
weight_t fgCalledCount; // count of the number of times this method was called
// This is derived from the profile data
// or is BB_UNITY_WEIGHT when we don't have profile data

#if defined(FEATURE_EH_FUNCLETS)
bool fgFuncletsCreated; // true if the funclet creation phase has been run
Expand Down Expand Up @@ -6015,7 +6010,6 @@ class Compiler
PhaseStatus fgComputeBlockAndEdgeWeights();
bool fgComputeMissingBlockWeights(weight_t* returnWeight);
bool fgComputeCalledCount(weight_t returnWeight);
PhaseStatus fgComputeEdgeWeights();

bool fgReorderBlocks(bool useProfile);

Expand Down Expand Up @@ -6826,7 +6820,6 @@ class Compiler
BasicBlock* optTryAdvanceLoopCompactionInsertionPoint(FlowGraphNaturalLoop* loop, BasicBlock* insertionPoint, BasicBlock* top, BasicBlock* bottom);
bool optCreatePreheader(FlowGraphNaturalLoop* loop);
void optSetWeightForPreheaderOrExit(FlowGraphNaturalLoop* loop, BasicBlock* block);
weight_t optEstimateEdgeLikelihood(BasicBlock* from, BasicBlock* to, bool* fromProfile);

bool optCanonicalizeExits(FlowGraphNaturalLoop* loop);
bool optCanonicalizeExit(FlowGraphNaturalLoop* loop, BasicBlock* exit);
Expand Down
29 changes: 6 additions & 23 deletions src/coreclr/jit/fgbasic.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,8 @@ void Compiler::fgInit()
/* We haven't yet computed the bbPreds lists */
fgPredsComputed = false;

/* We haven't yet computed the edge weight */
fgEdgeWeightsComputed = false;
fgHaveValidEdgeWeights = false;
fgSlopUsedInEdgeWeights = false;
fgRangeUsedInEdgeWeights = true;
fgCalledCount = BB_ZERO_WEIGHT;
/* We haven't yet computed block weights */
fgCalledCount = BB_ZERO_WEIGHT;

/* Initialize the basic block list */

Expand Down Expand Up @@ -5395,6 +5391,7 @@ BasicBlock* Compiler::fgConnectFallThrough(BasicBlock* bSrc, BasicBlock* bDst)
// Add a new block after bSrc which jumps to 'bDst'
jmpBlk = fgNewBBafter(BBJ_ALWAYS, bSrc, true);
FlowEdge* const oldEdge = bSrc->GetFalseEdge();

// Access the likelihood of oldEdge before
// it gets reset by SetTargetEdge below.
//
Expand All @@ -5406,29 +5403,15 @@ BasicBlock* Compiler::fgConnectFallThrough(BasicBlock* bSrc, BasicBlock* bDst)

// When adding a new jmpBlk we will set the bbWeight and bbFlags
//
if (fgHaveValidEdgeWeights && fgHaveProfileWeights())
if (fgHaveProfileWeights())
{
jmpBlk->bbWeight = (newEdge->edgeWeightMin() + newEdge->edgeWeightMax()) / 2;
if (bSrc->bbWeight == BB_ZERO_WEIGHT)
{
jmpBlk->bbWeight = BB_ZERO_WEIGHT;
}
jmpBlk->bbWeight = newEdge->getLikelyWeight();
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can use setBBProfileWeight here

jmpBlk->SetFlags(BBF_PROF_WEIGHT);

if (jmpBlk->bbWeight == BB_ZERO_WEIGHT)
{
jmpBlk->SetFlags(BBF_RUN_RARELY);
}

weight_t weightDiff = (newEdge->edgeWeightMax() - newEdge->edgeWeightMin());
weight_t slop = BasicBlock::GetSlopFraction(bSrc, bDst);
//
// If the [min/max] values for our edge weight is within the slop factor
// then we will set the BBF_PROF_WEIGHT flag for the block
//
if (weightDiff <= slop)
{
jmpBlk->SetFlags(BBF_PROF_WEIGHT);
}
}
else
{
Expand Down
71 changes: 20 additions & 51 deletions src/coreclr/jit/fgdiagnostic.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -28,26 +28,17 @@ void Compiler::fgPrintEdgeWeights()

printf(FMT_BB " ", bSrc->bbNum);

if (edge->edgeWeightMin() < BB_MAX_WEIGHT)
const weight_t weight = edge->getLikelyWeight();

if (weight < BB_MAX_WEIGHT)
{
printf("(%f", edge->edgeWeightMin());
printf("(%f)", weight);
}
else
{
printf("(MAX");
}
if (edge->edgeWeightMin() != edge->edgeWeightMax())
{
if (edge->edgeWeightMax() < BB_MAX_WEIGHT)
{
printf("..%f", edge->edgeWeightMax());
}
else
{
printf("..MAX");
}
printf("(MAX)");
}
printf(")");

if (edge->getNextPredEdge() != nullptr)
{
printf(", ");
Expand Down Expand Up @@ -735,7 +726,6 @@ bool Compiler::fgDumpFlowGraph(Phases phase, PhasePosition pos)
JITDUMP("Writing out flow graph %s phase %s\n", (pos == PhasePosition::PrePhase) ? "before" : "after",
PhaseNames[phase]);

bool validWeights = fgHaveValidEdgeWeights;
double weightDivisor = (double)BasicBlock::getCalledCount(this);
const char* escapedString;
const char* regionString = "NONE";
Expand Down Expand Up @@ -791,14 +781,6 @@ bool Compiler::fgDumpFlowGraph(Phases phase, PhasePosition pos)
{
fprintf(fgxFile, "\n hasLoops=\"true\"");
}
if (validWeights)
{
fprintf(fgxFile, "\n validEdgeWeights=\"true\"");
if (!fgSlopUsedInEdgeWeights && !fgRangeUsedInEdgeWeights)
{
fprintf(fgxFile, "\n exactEdgeWeights=\"true\"");
}
}
if (fgFirstColdBlock != nullptr)
{
fprintf(fgxFile, "\n firstColdBlock=\"%d\"", fgFirstColdBlock->bbNum);
Expand Down Expand Up @@ -1081,11 +1063,8 @@ bool Compiler::fgDumpFlowGraph(Phases phase, PhasePosition pos)
fprintf(fgxFile, " [");
}

if (validWeights)
{
weight_t edgeWeight = (edge->edgeWeightMin() + edge->edgeWeightMax()) / 2;
fprintf(fgxFile, "%slabel=\"%7.2f\"", sep, (double)edgeWeight / weightDivisor);
}
const weight_t edgeWeight = edge->getLikelyWeight();
fprintf(fgxFile, "%slabel=\"%7.2f\"", sep, (double)edgeWeight / weightDivisor);

fprintf(fgxFile, "];\n");
}
Expand All @@ -1106,32 +1085,22 @@ bool Compiler::fgDumpFlowGraph(Phases phase, PhasePosition pos)
fprintf(fgxFile, "\n switchDefault=\"true\"");
}
}
if (validWeights)
{
weight_t edgeWeight = (edge->edgeWeightMin() + edge->edgeWeightMax()) / 2;
fprintf(fgxFile, "\n weight=");
fprintfDouble(fgxFile, ((double)edgeWeight) / weightDivisor);

if (edge->edgeWeightMin() != edge->edgeWeightMax())
const weight_t edgeWeight = edge->getLikelyWeight();
fprintf(fgxFile, "\n weight=");
fprintfDouble(fgxFile, ((double)edgeWeight) / weightDivisor);

if (edgeWeight > 0)
{
if (edgeWeight < bSource->bbWeight)
{
fprintf(fgxFile, "\n minWeight=");
fprintfDouble(fgxFile, ((double)edge->edgeWeightMin()) / weightDivisor);
fprintf(fgxFile, "\n maxWeight=");
fprintfDouble(fgxFile, ((double)edge->edgeWeightMax()) / weightDivisor);
fprintf(fgxFile, "\n out=");
fprintfDouble(fgxFile, ((double)edgeWeight) / sourceWeightDivisor);
}

if (edgeWeight > 0)
if (edgeWeight < bTarget->bbWeight)
{
if (edgeWeight < bSource->bbWeight)
{
fprintf(fgxFile, "\n out=");
fprintfDouble(fgxFile, ((double)edgeWeight) / sourceWeightDivisor);
}
if (edgeWeight < bTarget->bbWeight)
{
fprintf(fgxFile, "\n in=");
fprintfDouble(fgxFile, ((double)edgeWeight) / targetWeightDivisor);
}
fprintf(fgxFile, "\n in=");
fprintfDouble(fgxFile, ((double)edgeWeight) / targetWeightDivisor);
}
}
}
Expand Down
36 changes: 0 additions & 36 deletions src/coreclr/jit/fgflow.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -191,42 +191,6 @@ FlowEdge* Compiler::fgAddRefPred(BasicBlock* block, BasicBlock* blockPred, FlowE
//
flow->setLikelihood(oldEdge->getLikelihood());
}

if (fgHaveValidEdgeWeights)
{
// We are creating an edge from blockPred to block
// and we have already computed the edge weights, so
// we will try to setup this new edge with valid edge weights.
//
if (oldEdge != nullptr)
{
// If our caller has given us the old edge weights
// then we will use them.
//
flow->setEdgeWeights(oldEdge->edgeWeightMin(), oldEdge->edgeWeightMax(), block);
}
else
{
// Set the max edge weight to be the minimum of block's or blockPred's weight
//
weight_t newWeightMax = min(block->bbWeight, blockPred->bbWeight);

// If we are inserting a conditional block the minimum weight is zero,
// otherwise it is the same as the edge's max weight.
if (blockPred->NumSucc() > 1)
{
flow->setEdgeWeights(BB_ZERO_WEIGHT, newWeightMax, block);
}
else
{
flow->setEdgeWeights(flow->edgeWeightMax(), newWeightMax, block);
}
}
}
else
{
flow->setEdgeWeights(BB_ZERO_WEIGHT, BB_MAX_WEIGHT, block);
}
}

// Pred list should (still) be ordered.
Expand Down
Loading
Loading