diff --git a/src/coreclr/jit/objectalloc.cpp b/src/coreclr/jit/objectalloc.cpp index 347520f1c70812..b88dc448ef679a 100644 --- a/src/coreclr/jit/objectalloc.cpp +++ b/src/coreclr/jit/objectalloc.cpp @@ -25,14 +25,14 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX // comp - compiler instance // // Notes: -// Runs only if Compiler::optMethodFlags has flag OMF_HAS_NEWOBJ set. +// Runs only if Compiler::optMethodFlags has flag OMF_HAS_NEWOBJ or OMF_HAS_NEWARR. // -// Builds a connection graph where nodes mostly represent local vars, -// showing how locals can assign values to one another. +// Builds a connection graph where nodes mostly represent gc typed local vars, +// showing how these locals can assign values to one another. // -// The graph also includes a few abstract node types: a node representing -// an unknown source of values, and (pseudo local) nodes representing -// assignments that only happen under particular conditions. +// The graph also includes a abstract node types: a node representing an unknown source of values, +// pseudo nodes representing assignments that only happen under particular conditions, +// and nodes representing fields of local structs. // ObjectAllocator::ObjectAllocator(Compiler* comp) : Phase(comp, PHASE_ALLOCATE_OBJECTS) @@ -41,16 +41,14 @@ ObjectAllocator::ObjectAllocator(Compiler* comp) , m_isR2R(comp->IsReadyToRun()) , m_bvCount(0) , m_bitVecTraits(BitVecTraits(comp->lvaCount, comp)) - , m_unknownSourceLocalNum(BAD_VAR_NUM) , m_unknownSourceIndex(BAD_VAR_NUM) , m_HeapLocalToStackLocalMap(comp->getAllocator(CMK_ObjectAllocator)) - , m_EnumeratorLocalToPseudoLocalMap(comp->getAllocator(CMK_ObjectAllocator)) + , m_EnumeratorLocalToPseudoIndexMap(comp->getAllocator(CMK_ObjectAllocator)) , m_CloneMap(comp->getAllocator(CMK_ObjectAllocator)) , m_nextLocalIndex(0) - , m_firstPseudoLocalNum(BAD_VAR_NUM) - , m_firstPseudoLocalIndex(BAD_VAR_NUM) - , m_numPseudoLocals(0) - , m_maxPseudoLocals(0) + , m_firstPseudoIndex(BAD_VAR_NUM) + , m_numPseudos(0) + , m_maxPseudos(0) , m_regionsToClone(0) , m_trackFields(false) { @@ -96,60 +94,20 @@ bool ObjectAllocator::IsTrackedLocal(unsigned lclNum) } //------------------------------------------------------------------------ -// HasIndex: see if a given local has a tracking index -// -// Arguments: -// lclNum -- local to query -// -// Returns: -// true if so -// -bool ObjectAllocator::HasIndex(unsigned lclNum) -{ - if (lclNum < comp->lvaCount) - { - LclVarDsc* const varDsc = comp->lvaGetDesc(lclNum); - return varDsc->lvTracked; - } - - if ((lclNum >= m_firstPseudoLocalNum) && (lclNum < m_bvCount)) - { - return true; - } - - return false; -} - -//------------------------------------------------------------------------ -// LocalToIndex: get the bit vector index for a local or pseudo-local +// LocalToIndex: get the bit vector index for a local // // Arguments: -// lclNum -- local var num or pseudo local var num +// lclNum -- local var num // // Returns: -// bvIndex to use +// bvIndex to use, or BAD_VAR_NUM if local is not tracked // unsigned ObjectAllocator::LocalToIndex(unsigned lclNum) { - unsigned result = BAD_VAR_NUM; - - if (lclNum < m_firstPseudoLocalNum) - { - assert(IsTrackedLocal(lclNum)); - LclVarDsc* const varDsc = comp->lvaGetDesc(lclNum); - result = varDsc->lvVarIndex; - } - else if (lclNum == m_unknownSourceLocalNum) - { - result = m_unknownSourceIndex; - } - else - { - result = m_firstPseudoLocalIndex + (lclNum - m_firstPseudoLocalNum); - } - + assert(IsTrackedLocal(lclNum)); + LclVarDsc* const varDsc = comp->lvaGetDesc(lclNum); + unsigned const result = varDsc->lvVarIndex; assert(result < m_bvCount); - return result; } @@ -160,23 +118,18 @@ unsigned ObjectAllocator::LocalToIndex(unsigned lclNum) // bvIndex -- bit vector index // // Returns: -// local num +// local num or BAD_VAR_NUM if index is not a tracked local var // unsigned ObjectAllocator::IndexToLocal(unsigned bvIndex) { assert(bvIndex < m_bvCount); unsigned result = BAD_VAR_NUM; - if (bvIndex < m_firstPseudoLocalIndex) + if (bvIndex < m_firstPseudoIndex) { result = comp->lvaTrackedToVarNum[bvIndex]; assert(IsTrackedLocal(result)); } - else - { - result = m_firstPseudoLocalNum + (bvIndex - m_firstPseudoLocalIndex); - } - return result; } @@ -192,10 +145,25 @@ unsigned ObjectAllocator::IndexToLocal(unsigned bvIndex) // void ObjectAllocator::DumpIndex(unsigned bvIndex) { - const unsigned lclNum = IndexToLocal(bvIndex); - const bool isLocalVar = (lclNum < m_firstPseudoLocalNum); - const bool isUnknown = (lclNum == m_unknownSourceLocalNum); - printf(" %c%02u", isUnknown ? 'U' : isLocalVar ? 'V' : 'P', lclNum); + if (bvIndex < m_firstPseudoIndex) + { + printf(" V%02u", IndexToLocal(bvIndex)); + return; + } + + if (bvIndex < m_unknownSourceIndex) + { + printf(" P%02u", bvIndex); + return; + } + + if (bvIndex == m_unknownSourceIndex) + { + printf(" U%02u", bvIndex); + return; + } + + printf(" ?%02u", bvIndex); } #endif @@ -279,10 +247,22 @@ PhaseStatus ObjectAllocator::DoPhase() // // Arguments: // lclNum - Escaping pointing local variable number - +// void ObjectAllocator::MarkLclVarAsEscaping(unsigned int lclNum) { const unsigned bvIndex = LocalToIndex(lclNum); + MarkIndexAsEscaping(bvIndex); +} + +//------------------------------------------------------------------------------ +// MarkIndexAsEscaping : Mark resource as escaping. +// +// +// Arguments: +// index - bv index for the resource +// +void ObjectAllocator::MarkIndexAsEscaping(unsigned int bvIndex) +{ BitVecOps::AddElemD(&m_bitVecTraits, m_EscapingPointers, bvIndex); } @@ -293,11 +273,23 @@ void ObjectAllocator::MarkLclVarAsEscaping(unsigned int lclNum) // // Arguments: // lclNum - Possibly stack-object-pointing local variable number - +// void ObjectAllocator::MarkLclVarAsPossiblyStackPointing(unsigned int lclNum) { const unsigned bvIndex = LocalToIndex(lclNum); - JITDUMP("Marking V%02u (0x%02x) as possibly stack-pointing\n", lclNum, bvIndex); + MarkIndexAsPossiblyStackPointing(bvIndex); +} + +//------------------------------------------------------------------------------ +// MarkIndexAsPossiblyStackPointing : Mark resource as possibly pointing +// to a stack-allocated object. +// +// +// Arguments: +// index - bv index for the resource +// +void ObjectAllocator::MarkIndexAsPossiblyStackPointing(unsigned int bvIndex) +{ BitVecOps::AddElemD(&m_bitVecTraits, m_PossiblyStackPointingPointers, bvIndex); } @@ -308,11 +300,23 @@ void ObjectAllocator::MarkLclVarAsPossiblyStackPointing(unsigned int lclNum) // // Arguments: // lclNum - Definitely stack-object-pointing local variable number - +// void ObjectAllocator::MarkLclVarAsDefinitelyStackPointing(unsigned int lclNum) { const unsigned bvIndex = LocalToIndex(lclNum); - JITDUMP("Marking V%02u (0x%02x) as definitely stack-pointing\n", lclNum, bvIndex); + MarkIndexAsDefinitelyStackPointing(bvIndex); +} + +//------------------------------------------------------------------------------ +// MarIndexAsDefinitelyStackPointing : Mark resource as definitely pointing +// to a stack-allocated object. +// +// +// Arguments: +// index - bv index for the resource +// +void ObjectAllocator::MarkIndexAsDefinitelyStackPointing(unsigned int bvIndex) +{ BitVecOps::AddElemD(&m_bitVecTraits, m_DefinitelyStackPointingPointers, bvIndex); } @@ -323,11 +327,24 @@ void ObjectAllocator::MarkLclVarAsDefinitelyStackPointing(unsigned int lclNum) // Arguments: // sourceLclNum - Local variable number of the edge source // targetLclNum - Local variable number of the edge target - +// void ObjectAllocator::AddConnGraphEdge(unsigned int sourceLclNum, unsigned int targetLclNum) { const unsigned sourceBvIndex = LocalToIndex(sourceLclNum); const unsigned targetBvIndex = LocalToIndex(targetLclNum); + AddConnGraphEdgeIndex(sourceBvIndex, targetBvIndex); +} + +//------------------------------------------------------------------------------ +// AddConnGraphEdgeIndex : Record that the source resource may point to the same set of objects +// as the set pointed to by target resource +// +// Arguments: +// sourceBvIndex - index of the edge source +// targetBvIndex - index of the edge target +// +void ObjectAllocator::AddConnGraphEdgeIndex(unsigned int sourceBvIndex, unsigned int targetBvIndex) +{ BitVecOps::AddElemD(&m_bitVecTraits, m_ConnGraphAdjacencyMatrix[sourceBvIndex], targetBvIndex); } @@ -346,11 +363,7 @@ void ObjectAllocator::PrepareAnalysis() // // If conditional escape analysis is enabled, we reserve the range [L...L+M-1] // for locals allocated during the conditional escape analysis expansions, - // where M is the maximum number of pseudo-vars. - // - // We reserve the range [L+M ... L+2M-1] for pseudo locals themselves. - // - // We reserve the singleton [L+2M] for the "unknown source" local + // where M is the maximum number of pseudos. // // In "bv" space // @@ -360,9 +373,9 @@ void ObjectAllocator::PrepareAnalysis() // // If conditional escape analysis is enabled, we reserve the range [N...N+M-1] // for locals allocated during the conditional escape analysis expansions, - // where N is the maximum number of pseudo-vars. + // where N is the maximum number of pseudos. // - // We reserve the range [N+M ... N+2M-1] for pseudo locals themselves. + // We reserve the range [N+M ... N+2M-1] for pseudos. // // We reserve the singleton [N+2M] for the "unknown source" local // @@ -427,8 +440,8 @@ void ObjectAllocator::PrepareAnalysis() if (inRange) { - JITDUMP("Enabling conditional escape analysis [%u pseudo-vars]\n", enumeratorLocalCount); - m_maxPseudoLocals = enumeratorLocalCount; + JITDUMP("Enabling conditional escape analysis [%u pseudos]\n", enumeratorLocalCount); + m_maxPseudos = enumeratorLocalCount; } else { @@ -437,7 +450,7 @@ void ObjectAllocator::PrepareAnalysis() } else { - JITDUMP("Not enabling conditional escape analysis [%u pseudo-vars]: %s\n", enumeratorLocalCount, + JITDUMP("Not enabling conditional escape analysis [%u pseudos]: %s\n", enumeratorLocalCount, enableConditionalEscape ? "OSR" : "disabled by config"); } } @@ -460,16 +473,15 @@ void ObjectAllocator::PrepareAnalysis() // When we clone to prevent conditional escape, we'll also create a new local // var that we will track. So we need to leave room for these vars. There can - // be as many of these as there are pseudo locals. + // be as many of these as there are pseudos. // - m_firstPseudoLocalNum = localCount + m_maxPseudoLocals; // L + M, per above - m_firstPseudoLocalIndex = bvNext + m_maxPseudoLocals; // N, per above - bvNext += 2 * m_maxPseudoLocals; + const unsigned maxTrackedLclNum = localCount + m_maxPseudos; + m_firstPseudoIndex = bvNext + m_maxPseudos; // N, per above + bvNext += 2 * m_maxPseudos; - // A local representing an unknown source of values + // A bv index for an unknown source of values // - m_unknownSourceLocalNum = m_firstPseudoLocalNum + m_maxPseudoLocals; - m_unknownSourceIndex = bvNext; + m_unknownSourceIndex = bvNext; bvNext++; // Now set up the BV traits. @@ -480,9 +492,9 @@ void ObjectAllocator::PrepareAnalysis() // Create the reverse mapping from bvIndex to local var index // (leave room for locals we may allocate) // - if (comp->lvaTrackedToVarNumSize < m_firstPseudoLocalNum) + if (comp->lvaTrackedToVarNumSize < maxTrackedLclNum) { - comp->lvaTrackedToVarNumSize = m_firstPseudoLocalNum; + comp->lvaTrackedToVarNumSize = maxTrackedLclNum; comp->lvaTrackedToVarNum = new (comp->getAllocator(CMK_LvaTable)) unsigned[comp->lvaTrackedToVarNumSize]; } @@ -502,21 +514,17 @@ void ObjectAllocator::PrepareAnalysis() if (m_nextLocalIndex > 0) { JITDUMP("\nLocal var range [%02u...%02u]\n", 0, localCount - 1); - if (m_maxPseudoLocals > 0) + if (m_maxPseudos > 0) { - JITDUMP("Enumerator var range [%02u...%02u]\n", localCount, localCount + m_maxPseudoLocals - 1); - JITDUMP("Pseudo var range [%02u...%02u]\n", m_firstPseudoLocalNum, - m_firstPseudoLocalNum + m_maxPseudoLocals - 1); + JITDUMP("Enumerator var range [%02u...%02u]\n", localCount, localCount + m_maxPseudos - 1); } - JITDUMP("Unknown var range [%02u...%02u]\n", m_unknownSourceLocalNum, m_unknownSourceLocalNum); JITDUMP("\nLocal var bv range [%02u...%02u]\n", 0, m_nextLocalIndex - 1); - if (m_maxPseudoLocals > 0) + if (m_maxPseudos > 0) { - JITDUMP("Enumerator var bv range [%02u...%02u]\n", m_nextLocalIndex, - m_nextLocalIndex + m_maxPseudoLocals - 1); - JITDUMP("Pseudo var bv range [%02u...%02u]\n", m_nextLocalIndex + m_maxPseudoLocals, - m_nextLocalIndex + 2 * m_maxPseudoLocals - 1); + JITDUMP("Enumerator var bv range [%02u...%02u]\n", m_nextLocalIndex, m_nextLocalIndex + m_maxPseudos - 1); + JITDUMP("Pseudo var bv range [%02u...%02u]\n", m_nextLocalIndex + m_maxPseudos, + m_nextLocalIndex + 2 * m_maxPseudos - 1); } JITDUMP("Unknown var bv range [%02u...%02u]\n", m_unknownSourceIndex, m_unknownSourceIndex); } @@ -540,13 +548,18 @@ void ObjectAllocator::DoAnalysis() // If we are doing conditional escape analysis, we also need to compute dominance. // - if (CanHavePseudoLocals()) + if (CanHavePseudos()) { assert(comp->m_dfsTree != nullptr); assert(comp->m_domTree == nullptr); comp->m_domTree = FlowGraphDominatorTree::Build(comp->m_dfsTree); } + for (unsigned int i = 0; i < m_bvCount; i++) + { + m_ConnGraphAdjacencyMatrix[i] = BitVecOps::MakeEmpty(&m_bitVecTraits); + } + MarkEscapingVarsAndBuildConnGraph(); ComputeEscapingNodes(&m_bitVecTraits, m_EscapingPointers); } @@ -674,7 +687,8 @@ void ObjectAllocator::MarkEscapingVarsAndBuildConnGraph() // Add a connection to the unknown source. // JITDUMP("V%02u value unknown at [%06u]\n", lclNum, m_compiler->dspTreeID(tree)); - m_allocator->AddConnGraphEdge(lclNum, m_allocator->m_unknownSourceLocalNum); + m_allocator->AddConnGraphEdgeIndex(m_allocator->LocalToIndex(lclNum), + m_allocator->m_unknownSourceIndex); } } } @@ -691,21 +705,20 @@ void ObjectAllocator::MarkEscapingVarsAndBuildConnGraph() continue; } - LclVarDsc* const lclDsc = comp->lvaGetDesc(lclNum); - const unsigned bvIndex = LocalToIndex(lclNum); - m_ConnGraphAdjacencyMatrix[bvIndex] = BitVecOps::MakeEmpty(&m_bitVecTraits); + LclVarDsc* const lclDsc = comp->lvaGetDesc(lclNum); + const unsigned bvIndex = LocalToIndex(lclNum); if (lclDsc->IsAddressExposed()) { JITDUMP(" V%02u is address exposed\n", lclNum); - MarkLclVarAsEscaping(lclNum); + MarkIndexAsEscaping(bvIndex); continue; } if (lclNum == comp->info.compRetBuffArg) { JITDUMP(" V%02u is retbuff\n", lclNum); - MarkLclVarAsEscaping(lclNum); + MarkIndexAsEscaping(bvIndex); continue; } @@ -716,7 +729,7 @@ void ObjectAllocator::MarkEscapingVarsAndBuildConnGraph() if (lclDsc->lvIsParam && lclDsc->lvIsImplicitByRef) { JITDUMP(" V%02u is an implicit byref param\n", lclNum); - MarkLclVarAsEscaping(lclNum); + MarkIndexAsEscaping(bvIndex); continue; } #endif @@ -726,17 +739,11 @@ void ObjectAllocator::MarkEscapingVarsAndBuildConnGraph() // if (lclDsc->lvIsParam || lclDsc->lvIsOSRLocal) { - AddConnGraphEdge(lclNum, m_unknownSourceLocalNum); + AddConnGraphEdgeIndex(bvIndex, m_unknownSourceIndex); } } - for (unsigned int p = 0; p < m_maxPseudoLocals; p++) - { - m_ConnGraphAdjacencyMatrix[p + m_firstPseudoLocalIndex] = BitVecOps::MakeEmpty(&m_bitVecTraits); - } - - m_ConnGraphAdjacencyMatrix[m_unknownSourceIndex] = BitVecOps::MakeEmpty(&m_bitVecTraits); - MarkLclVarAsEscaping(m_unknownSourceLocalNum); + MarkIndexAsEscaping(m_unknownSourceIndex); // We should have computed the DFS tree already. // @@ -819,7 +826,7 @@ void ObjectAllocator::ComputeEscapingNodes(BitVecTraits* bitVecTraits, BitVec& e computeClosure(); - if (m_numPseudoLocals > 0) + if (m_numPseudos > 0) { bool newEscapes = AnalyzeIfCloningCanPreventEscape(bitVecTraits, escapingNodes, escapingNodesToProcess); if (newEscapes) @@ -851,34 +858,27 @@ void ObjectAllocator::ComputeStackObjectPointers(BitVecTraits* bitVecTraits) { JITDUMP("\n---- computing stack pointing locals, pass %u\n", pass++); changed = false; - for (unsigned int lclNum = 0; lclNum < comp->lvaCount; ++lclNum) + for (unsigned int index = 0; index < m_bvCount; index++) { - if (!IsTrackedLocal(lclNum)) - { - continue; - } - - const unsigned lclIndex = LocalToIndex(lclNum); - - if (!MayLclVarPointToStack(lclNum) && + if (!MayIndexPointToStack(index) && !BitVecOps::IsEmptyIntersection(bitVecTraits, m_PossiblyStackPointingPointers, - m_ConnGraphAdjacencyMatrix[lclIndex])) + m_ConnGraphAdjacencyMatrix[index])) { // We discovered a new pointer that may point to the stack. - JITDUMPEXEC(DumpIndex(lclIndex)); + JITDUMPEXEC(DumpIndex(index)); JITDUMP(" may point to the stack\n"); - MarkLclVarAsPossiblyStackPointing(lclNum); + MarkIndexAsPossiblyStackPointing(index); changed = true; } - if (!BitVecOps::IsMember(bitVecTraits, possiblyHeapPointingPointers, lclIndex) && + if (!BitVecOps::IsMember(bitVecTraits, possiblyHeapPointingPointers, index) && !BitVecOps::IsEmptyIntersection(bitVecTraits, possiblyHeapPointingPointers, - m_ConnGraphAdjacencyMatrix[lclIndex])) + m_ConnGraphAdjacencyMatrix[index])) { // We discovered a new pointer that may point to the heap. - JITDUMPEXEC(DumpIndex(lclIndex)); + JITDUMPEXEC(DumpIndex(index)); JITDUMP(" may point to the heap\n"); - BitVecOps::AddElemD(bitVecTraits, possiblyHeapPointingPointers, lclIndex); + BitVecOps::AddElemD(bitVecTraits, possiblyHeapPointingPointers, index); changed = true; } } @@ -902,10 +902,10 @@ void ObjectAllocator::ComputeStackObjectPointers(BitVecTraits* bitVecTraits) printf("Definitely stack-pointing locals:"); { BitVecOps::Iter iter(bitVecTraits, m_DefinitelyStackPointingPointers); - unsigned lclIndex = 0; - while (iter.NextElem(&lclIndex)) + unsigned index = 0; + while (iter.NextElem(&index)) { - DumpIndex(lclIndex); + DumpIndex(index); } printf("\n"); } @@ -913,12 +913,12 @@ void ObjectAllocator::ComputeStackObjectPointers(BitVecTraits* bitVecTraits) printf("Possibly stack-pointing locals:"); { BitVecOps::Iter iter(bitVecTraits, m_PossiblyStackPointingPointers); - unsigned lclIndex = 0; - while (iter.NextElem(&lclIndex)) + unsigned index = 0; + while (iter.NextElem(&index)) { - if (!BitVecOps::IsMember(bitVecTraits, m_DefinitelyStackPointingPointers, lclIndex)) + if (!BitVecOps::IsMember(bitVecTraits, m_DefinitelyStackPointingPointers, index)) { - DumpIndex(lclIndex); + DumpIndex(index); } } printf("\n"); @@ -1290,11 +1290,11 @@ bool ObjectAllocator::MorphAllocObjNodes() // and the enumeratorLocal we already allocated. This is needed because we do early rewriting // in the conditional clone. // - unsigned pseudoLocal = BAD_VAR_NUM; - if (m_EnumeratorLocalToPseudoLocalMap.TryGetValue(lclNum, &pseudoLocal)) + unsigned pseudoIndex = BAD_VAR_NUM; + if (m_EnumeratorLocalToPseudoIndexMap.TryGetValue(lclNum, &pseudoIndex)) { CloneInfo* info = nullptr; - if (m_CloneMap.Lookup(pseudoLocal, &info)) + if (m_CloneMap.Lookup(pseudoIndex, &info)) { if (info->m_willClone) { @@ -1328,7 +1328,7 @@ bool ObjectAllocator::MorphAllocObjNodes() if (IsTrackedLocal(lclNum)) { - AddConnGraphEdge(lclNum, m_unknownSourceLocalNum); + AddConnGraphEdgeIndex(LocalToIndex(lclNum), m_unknownSourceIndex); } } } @@ -1886,7 +1886,7 @@ bool ObjectAllocator::CanLclVarEscapeViaParentStack(ArrayStack* parent // // In particular it might be tempting to look for references in uncatchable BBJ_THROWs or similar // and enable a kind of "partial escape analysis" where we copy from stack to heap just before the - // point of escape. We would have to add pseudo-locals for this like we do for GDV, but we wouldn't + // point of escape. We would have to add pseudos for this like we do for GDV, but we wouldn't // necessarily need to do the predicate analysis or cloning. // if (isEnumeratorLocal) @@ -2470,14 +2470,14 @@ void ObjectAllocator::RewriteUses() // // Notes: // During our analysis we have may have noted conditionally escaping objects -// and var references and connected them to a pseduolocal, along with information +// and var references and connected them to a pseduo, along with information // about how we could clone blocks to ensure that the object could be stack allocated. // // The current assumption is that these nodes do not escape, but to ensure // that we must be able to clone the code and remove the potential for escape // -// So, we verify for each case that we can clone; if not, mark we the pseudolocal -// as escaping. If any pseudo local now escapes, we return true so that the main +// So, we verify for each case that we can clone; if not, mark we the Pseudo +// as escaping. If any pseudo now escapes, we return true so that the main // analysis can update its closure. // // We may choose not to clone a candiate for several reasons: @@ -2491,38 +2491,41 @@ bool ObjectAllocator::AnalyzeIfCloningCanPreventEscape(BitVecTraits* bitVecTrait { bool newEscapes = false; - for (unsigned p = 0; p < m_numPseudoLocals; p++) + for (unsigned p = 0; p < m_numPseudos; p++) { - unsigned const pseudoLocal = p + m_firstPseudoLocalNum; + unsigned const pseudoIndex = p + m_firstPseudoIndex; bool canClone = true; CloneInfo* info = nullptr; - const bool hasInfo = m_CloneMap.Lookup(pseudoLocal, &info); + const bool hasInfo = m_CloneMap.Lookup(pseudoIndex, &info); if (!hasInfo) { - // We never found any conditional allocation attached to this pseudoLocal. + // We never found any conditional allocation attached to this pseudoIndex. // - JITDUMP(" P%02u has no guard info\n", pseudoLocal); + JITDUMPEXEC(DumpIndex(pseudoIndex)); + JITDUMP(" has no guard info\n"); canClone = false; break; } - // See what locals were "assigned" to the pseudo local. + // See what locals were "assigned" to the pseudo. // - BitVec pseudoLocalAdjacencies = m_ConnGraphAdjacencyMatrix[LocalToIndex(pseudoLocal)]; + BitVec PseudoAdjacencies = m_ConnGraphAdjacencyMatrix[pseudoIndex]; // If we found an allocation but didn't find any conditionally escaping uses, then cloning is of no use // - if (BitVecOps::IsEmpty(bitVecTraits, pseudoLocalAdjacencies)) + if (BitVecOps::IsEmpty(bitVecTraits, PseudoAdjacencies)) { - JITDUMP(" No conditionally escaping uses under P%02u, so no reason to clone\n", pseudoLocal); + JITDUMP(" No conditionally escaping uses under"); + JITDUMPEXEC(DumpIndex(pseudoIndex)); + JITDUMP(", so no reason to clone\n"); canClone = false; break; } // Check if each conditionally escaping local escapes on its own; if so cloning is of no use // - BitVecOps::Iter iterator(bitVecTraits, pseudoLocalAdjacencies); + BitVecOps::Iter iterator(bitVecTraits, PseudoAdjacencies); unsigned lclNumIndex = BAD_VAR_NUM; while (canClone && iterator.NextElem(&lclNumIndex)) { @@ -2531,7 +2534,10 @@ bool ObjectAllocator::AnalyzeIfCloningCanPreventEscape(BitVecTraits* bitVecTrait // The enumerator var or a related var had escaping uses somewhere in the method, // not under a failing GDV or any GDV. // - JITDUMP(" V%02u escapes independently of P%02u\n", IndexToLocal(lclNumIndex), pseudoLocal); + JITDUMPEXEC(DumpIndex(lclNumIndex)); + JITDUMP(" escapes independently of", IndexToLocal(lclNumIndex)); + JITDUMPEXEC(DumpIndex(pseudoIndex)); + JITDUMP("\n"); canClone = false; break; } @@ -2545,7 +2551,12 @@ bool ObjectAllocator::AnalyzeIfCloningCanPreventEscape(BitVecTraits* bitVecTrait { if (BitVecOps::IsMember(bitVecTraits, escapingNodes, LocalToIndex(v))) { - JITDUMP(" alloc temp V%02u escapes independently of P%02u\n", v, pseudoLocal) + JITDUMP(" alloc temp"); + JITDUMPEXEC(DumpIndex(v)); + JITDUMP(" escapes independently of", IndexToLocal(lclNumIndex)); + JITDUMPEXEC(DumpIndex(pseudoIndex)); + JITDUMP("\n"); + canClone = false; break; } @@ -2557,7 +2568,8 @@ bool ObjectAllocator::AnalyzeIfCloningCanPreventEscape(BitVecTraits* bitVecTrait // We may be able to clone and specialize the enumerator uses to ensure // that the allocated enumerator does not escape. // - JITDUMP(" P%02u is guarding the escape of V%02u\n", pseudoLocal, info->m_local); + JITDUMPEXEC(DumpIndex(pseudoIndex)); + JITDUMP(" is guarding the escape of V%02u\n", info->m_local); if (info->m_allocTemps != nullptr) { JITDUMP(" along with "); @@ -2598,15 +2610,20 @@ bool ObjectAllocator::AnalyzeIfCloningCanPreventEscape(BitVecTraits* bitVecTrait // if (canClone) { - JITDUMP("\n*** Can prevent escape under P%02u via cloning ***\n", pseudoLocal); + JITDUMP("\n*** Can prevent escape under"); + JITDUMPEXEC(DumpIndex(pseudoIndex)); + JITDUMP(" via cloning ***\n"); + info->m_willClone = true; m_regionsToClone++; } else { - JITDUMP(" not optimizing, so will mark P%02u as escaping\n", pseudoLocal); - MarkLclVarAsEscaping(pseudoLocal); - BitVecOps::AddElemD(bitVecTraits, escapingNodesToProcess, LocalToIndex(pseudoLocal)); + JITDUMP(" not optimizing, so will mark"); + JITDUMPEXEC(DumpIndex(pseudoIndex)); + JITDUMP(" as escaping\n"); + MarkIndexAsEscaping(pseudoIndex); + BitVecOps::AddElemD(bitVecTraits, escapingNodesToProcess, pseudoIndex); newEscapes = true; } } @@ -2615,18 +2632,22 @@ bool ObjectAllocator::AnalyzeIfCloningCanPreventEscape(BitVecTraits* bitVecTrait } //------------------------------------------------------------------------------ -// NewPseudoLocal: return index of a new pseudo local. +// NewPseudoIndex: return index of a new pseudo. // // Returns: // index to use, or BAD_VAR_NUM if no more indices are available. // -unsigned ObjectAllocator::NewPseudoLocal() +unsigned ObjectAllocator::NewPseudoIndex() { unsigned result = BAD_VAR_NUM; - if (m_numPseudoLocals < m_maxPseudoLocals) + if (m_numPseudos >= m_maxPseudos) + { + assert(!"unexpected number of pseudos"); + } + else { - result = m_firstPseudoLocalNum + m_numPseudoLocals; - m_numPseudoLocals++; + result = m_firstPseudoIndex + m_numPseudos; + m_numPseudos++; } return result; } @@ -2835,12 +2856,12 @@ GenTree* ObjectAllocator::IsGuard(BasicBlock* block, GuardInfo* info) // bool ObjectAllocator::CheckForGuardedUse(BasicBlock* block, GenTree* tree, unsigned lclNum) { - // Find pseudo local... + // Find pseudo... // - unsigned pseudoLocal = BAD_VAR_NUM; - if (!m_EnumeratorLocalToPseudoLocalMap.TryGetValue(lclNum, &pseudoLocal)) + unsigned pseudoIndex = BAD_VAR_NUM; + if (!m_EnumeratorLocalToPseudoIndexMap.TryGetValue(lclNum, &pseudoIndex)) { - JITDUMP("... no pseudo local?\n"); + JITDUMP("... no pseudo?\n"); return false; } @@ -2853,10 +2874,10 @@ bool ObjectAllocator::CheckForGuardedUse(BasicBlock* block, GenTree* tree, unsig return false; } - // Find the GDV guard for the pseudo-local + // Find the GDV guard for the pseudo // CloneInfo* pseudoGuardInfo; - if (!m_CloneMap.Lookup(pseudoLocal, &pseudoGuardInfo)) + if (!m_CloneMap.Lookup(pseudoIndex, &pseudoGuardInfo)) { JITDUMP("... under non-gdv guard?\n"); return false; @@ -2866,14 +2887,16 @@ bool ObjectAllocator::CheckForGuardedUse(BasicBlock* block, GenTree* tree, unsig // if ((info.m_local == lclNum) && (pseudoGuardInfo->m_local == lclNum) && (info.m_type == pseudoGuardInfo->m_type)) { - // If so, track this as an assignment pseudoLocal = ... + // If so, track this as an assignment pseudoIndex = ... // // Later if we don't clone and split off the failing GDV paths, - // we will mark pseudoLocal as escaped, and that will lead + // we will mark pseudoIndex as escaped, and that will lead // to lclNum escaping as well. // - JITDUMP("... under GDV; tracking via pseudo-local P%02u\n", pseudoLocal); - AddConnGraphEdge(pseudoLocal, lclNum); + JITDUMP("... under GDV; tracking via pseudo index"); + JITDUMPEXEC(DumpIndex(pseudoIndex)); + JITDUMP("\n") + AddConnGraphEdgeIndex(pseudoIndex, LocalToIndex(lclNum)); return true; } @@ -2904,7 +2927,7 @@ void ObjectAllocator::CheckForGuardedAllocationOrCopy(BasicBlock* block, GenTree* const tree = *use; assert(tree->OperIsLocalStore()); - if (!CanHavePseudoLocals()) + if (!CanHavePseudos()) { // We didn't flag any allocations of interest during importation, // so there is nothing to do here. @@ -2951,11 +2974,11 @@ void ObjectAllocator::CheckForGuardedAllocationOrCopy(BasicBlock* block, if (CanAllocateLclVarOnStack(enumeratorLocal, clsHnd, OAT_NEWOBJ, length, &size, &reason, /* preliminaryCheck */ true)) { - // We are going to conditionally track accesses to the enumerator local via a pseudo local. + // We are going to conditionally track accesses to the enumerator local via a pseudo. // - const unsigned pseudoLocal = NewPseudoLocal(); - assert(pseudoLocal != BAD_VAR_NUM); - bool added = m_EnumeratorLocalToPseudoLocalMap.AddOrUpdate(enumeratorLocal, pseudoLocal); + const unsigned pseudoIndex = NewPseudoIndex(); + assert(pseudoIndex != BAD_VAR_NUM); + bool added = m_EnumeratorLocalToPseudoIndexMap.AddOrUpdate(enumeratorLocal, pseudoIndex); if (!added) { @@ -2977,16 +3000,18 @@ void ObjectAllocator::CheckForGuardedAllocationOrCopy(BasicBlock* block, CloneInfo* info = new (alloc) CloneInfo(); info->m_local = enumeratorLocal; info->m_type = clsHnd; - info->m_pseudoLocal = pseudoLocal; + info->m_pseudoIndex = pseudoIndex; info->m_appearanceMap = new (alloc) EnumeratorVarMap(alloc); info->m_allocBlock = block; info->m_allocStmt = stmt; info->m_allocTree = data; info->m_domBlock = controllingGDV.m_block; - m_CloneMap.Set(pseudoLocal, info); + m_CloneMap.Set(pseudoIndex, info); - JITDUMP("Enumerator allocation [%06u]: will track accesses to V%02u guarded by type %s via P%02u\n", - comp->dspTreeID(data), enumeratorLocal, comp->eeGetClassName(clsHnd), pseudoLocal); + JITDUMP("Enumerator allocation [%06u]: will track accesses to V%02u guarded by type %s via", + comp->dspTreeID(data), enumeratorLocal, comp->eeGetClassName(clsHnd)); + JITDUMPEXEC(DumpIndex(pseudoIndex)); + JITDUMP("\n"); // If this is not a direct assignment to the enumerator var we also need to // track the temps that will appear in between. Later we will rewrite these @@ -3056,16 +3081,16 @@ void ObjectAllocator::CheckForGuardedAllocationOrCopy(BasicBlock* block, // bool ObjectAllocator::CheckForEnumeratorUse(unsigned lclNum, unsigned dstLclNum) { - unsigned pseudoLocal = BAD_VAR_NUM; + unsigned pseudoIndex = BAD_VAR_NUM; - if (m_EnumeratorLocalToPseudoLocalMap.TryGetValue(dstLclNum, &pseudoLocal)) + if (m_EnumeratorLocalToPseudoIndexMap.TryGetValue(dstLclNum, &pseudoIndex)) { // We already knew dstLclNum was a potential copy // return true; } - if (!m_EnumeratorLocalToPseudoLocalMap.TryGetValue(lclNum, &pseudoLocal)) + if (!m_EnumeratorLocalToPseudoIndexMap.TryGetValue(lclNum, &pseudoIndex)) { // lclNum is not a potential source // @@ -3073,7 +3098,7 @@ bool ObjectAllocator::CheckForEnumeratorUse(unsigned lclNum, unsigned dstLclNum) } CloneInfo* info = nullptr; - if (!m_CloneMap.Lookup(pseudoLocal, &info)) + if (!m_CloneMap.Lookup(pseudoIndex, &info)) { // We aren't interested in locals under this guard // @@ -3082,11 +3107,13 @@ bool ObjectAllocator::CheckForEnumeratorUse(unsigned lclNum, unsigned dstLclNum) // lclNum is an interesting enumerator var, so now so is dstLclNum. // - const bool added = m_EnumeratorLocalToPseudoLocalMap.AddOrUpdate(dstLclNum, pseudoLocal); + const bool added = m_EnumeratorLocalToPseudoIndexMap.AddOrUpdate(dstLclNum, pseudoIndex); assert(added); - JITDUMP("Enumerator allocation: will also track accesses to V%02u via P%02u\n", dstLclNum, pseudoLocal); + JITDUMP("Enumerator allocation: will also track accesses to V%02u via", dstLclNum); + JITDUMPEXEC(DumpIndex(pseudoIndex)); + JITDUMP("\n"); if (info->m_allocTemps == nullptr) { @@ -3110,14 +3137,14 @@ bool ObjectAllocator::CheckForEnumeratorUse(unsigned lclNum, unsigned dstLclNum) // void ObjectAllocator::RecordAppearance(unsigned lclNum, BasicBlock* block, Statement* stmt, GenTree** use) { - unsigned pseudoLocal = BAD_VAR_NUM; - if (!m_EnumeratorLocalToPseudoLocalMap.TryGetValue(lclNum, &pseudoLocal)) + unsigned Pseudo = BAD_VAR_NUM; + if (!m_EnumeratorLocalToPseudoIndexMap.TryGetValue(lclNum, &Pseudo)) { return; } CloneInfo* info; - if (!m_CloneMap.Lookup(pseudoLocal, &info)) + if (!m_CloneMap.Lookup(Pseudo, &info)) { return; } @@ -3198,8 +3225,11 @@ bool ObjectAllocator::CloneOverlaps(CloneInfo* info) continue; } - JITDUMP("Cloned blocks for P%02u overlap with those for P%02u; unable to clone\n", info->m_pseudoLocal, - c->m_pseudoLocal); + JITDUMP("Cloned blocks for"); + JITDUMPEXEC(DumpIndex(info->m_pseudoIndex)); + JITDUMP(" overlap with those for"); + JITDUMPEXEC(DumpIndex(c->m_pseudoIndex)); + JITDUMP(" unable to clone\n"); overlaps = true; break; @@ -3235,7 +3265,9 @@ bool ObjectAllocator::ShouldClone(CloneInfo* info) unsigned blockSize = 0; if (block->ComplexityExceeds(comp, slack, &blockSize)) { - JITDUMP("Rejecting P%02u cloning: exceeds size limit %u\n", info->m_pseudoLocal, sizeLimit); + JITDUMP("Rejecting"); + JITDUMPEXEC(DumpIndex(info->m_pseudoIndex)); + JITDUMP(" cloning: exceeds size limit %u\n", sizeLimit); return false; } size += blockSize; @@ -3243,7 +3275,9 @@ bool ObjectAllocator::ShouldClone(CloneInfo* info) // TODO: some kind of profile check... // - JITDUMP("Accepting P%02u cloning: size %u does not exceed size limit %u\n", info->m_pseudoLocal, size, sizeLimit); + JITDUMP("Accepting"); + JITDUMPEXEC(DumpIndex(info->m_pseudoIndex)); + JITDUMP(" cloning: size %u does not exceed size limit %u\n", size, sizeLimit); return true; } @@ -3905,8 +3939,8 @@ void ObjectAllocator::CloneAndSpecialize(CloneInfo* info) comp->lvaTrackedToVarNum[newEnumeratorDsc->lvVarIndex] = newEnumeratorLocal; m_ConnGraphAdjacencyMatrix[newEnumeratorDsc->lvVarIndex] = BitVecOps::MakeEmpty(&m_bitVecTraits); m_nextLocalIndex++; - assert(m_maxPseudoLocals > 0); - assert(newEnumeratorDsc->lvVarIndex < m_firstPseudoLocalIndex); + assert(m_maxPseudos > 0); + assert(newEnumeratorDsc->lvVarIndex < m_firstPseudoIndex); JITDUMP("Tracking V%02u via 0x%02x\n", newEnumeratorLocal, newEnumeratorDsc->lvVarIndex); diff --git a/src/coreclr/jit/objectalloc.h b/src/coreclr/jit/objectalloc.h index b3d07d7eb060cc..599b556db75d02 100644 --- a/src/coreclr/jit/objectalloc.h +++ b/src/coreclr/jit/objectalloc.h @@ -75,8 +75,8 @@ struct CloneInfo : public GuardInfo m_blocks = BitVecOps::UninitVal(); } - // Pseudo-local tracking conditinal escapes - unsigned m_pseudoLocal = BAD_VAR_NUM; + // Pseudo-local tracking conditional escapes + unsigned m_pseudoIndex = BAD_VAR_NUM; // Local allocated for the address of the enumerator unsigned m_enumeratorLocal = BAD_VAR_NUM; @@ -128,7 +128,6 @@ class ObjectAllocator final : public Phase bool m_isR2R; unsigned m_bvCount; BitVecTraits m_bitVecTraits; - unsigned m_unknownSourceLocalNum; unsigned m_unknownSourceIndex; BitVec m_EscapingPointers; // We keep the set of possibly-stack-pointing pointers as a superset of the set of @@ -140,13 +139,12 @@ class ObjectAllocator final : public Phase unsigned int m_StackAllocMaxSize; // Info for conditionally-escaping locals - LocalToLocalMap m_EnumeratorLocalToPseudoLocalMap; + LocalToLocalMap m_EnumeratorLocalToPseudoIndexMap; CloneMap m_CloneMap; unsigned m_nextLocalIndex; - unsigned m_firstPseudoLocalNum; - unsigned m_firstPseudoLocalIndex; - unsigned m_numPseudoLocals; - unsigned m_maxPseudoLocals; + unsigned m_firstPseudoIndex; + unsigned m_numPseudos; + unsigned m_maxPseudos; unsigned m_regionsToClone; // Struct fields @@ -172,19 +170,24 @@ class ObjectAllocator final : public Phase private: bool IsTrackedType(var_types type); bool IsTrackedLocal(unsigned lclNum); - bool HasIndex(unsigned lclNum); unsigned LocalToIndex(unsigned lclNum); unsigned IndexToLocal(unsigned bvIndex); bool CanLclVarEscape(unsigned int lclNum); void MarkLclVarAsPossiblyStackPointing(unsigned int lclNum); + void MarkIndexAsPossiblyStackPointing(unsigned int index); void MarkLclVarAsDefinitelyStackPointing(unsigned int lclNum); + void MarkIndexAsDefinitelyStackPointing(unsigned int index); bool MayLclVarPointToStack(unsigned int lclNum); bool DoesLclVarPointToStack(unsigned int lclNum); + bool MayIndexPointToStack(unsigned int index); + bool DoesIndexPointToStack(unsigned int index); void PrepareAnalysis(); void DoAnalysis(); void MarkLclVarAsEscaping(unsigned int lclNum); + void MarkIndexAsEscaping(unsigned int lclNum); void MarkEscapingVarsAndBuildConnGraph(); void AddConnGraphEdge(unsigned int sourceLclNum, unsigned int targetLclNum); + void AddConnGraphEdgeIndex(unsigned int sourceIndex, unsigned int targetIndex); void ComputeEscapingNodes(BitVecTraits* bitVecTraits, BitVec& escapingNodes); void ComputeStackObjectPointers(BitVecTraits* bitVecTraits); bool MorphAllocObjNodes(); @@ -212,11 +215,11 @@ class ObjectAllocator final : public Phase bool CheckForEnumeratorUse(unsigned lclNum, unsigned dstLclNum); bool IsGuarded(BasicBlock* block, GenTree* tree, GuardInfo* info, bool testOutcome); GenTree* IsGuard(BasicBlock* block, GuardInfo* info); - unsigned NewPseudoLocal(); + unsigned NewPseudoIndex(); - bool CanHavePseudoLocals() + bool CanHavePseudos() { - return (m_maxPseudoLocals > 0); + return (m_maxPseudos > 0); } void RecordAppearance(unsigned lclNum, BasicBlock* block, Statement* stmt, GenTree** use); @@ -272,7 +275,7 @@ inline void ObjectAllocator::EnableObjectStackAllocation() inline bool ObjectAllocator::CanLclVarEscape(unsigned int lclNum) { - if (!HasIndex(lclNum)) + if (!IsTrackedLocal(lclNum)) { return true; } @@ -281,6 +284,22 @@ inline bool ObjectAllocator::CanLclVarEscape(unsigned int lclNum) return BitVecOps::IsMember(&m_bitVecTraits, m_EscapingPointers, bvIndex); } +//------------------------------------------------------------------------ +// MayIndexPointToStack: Returns true iff the resource described by index may +// point to a stack-allocated object +// +// Arguments: +// index - bv index +// +// Return Value: +// Returns true if so. +// +inline bool ObjectAllocator::MayIndexPointToStack(unsigned int index) +{ + assert(m_AnalysisDone); + return BitVecOps::IsMember(&m_bitVecTraits, m_PossiblyStackPointingPointers, index); +} + //------------------------------------------------------------------------ // MayLclVarPointToStack: Returns true iff local variable may // point to a stack-allocated object @@ -290,18 +309,33 @@ inline bool ObjectAllocator::CanLclVarEscape(unsigned int lclNum) // // Return Value: // Returns true iff local variable may point to a stack-allocated object - +// inline bool ObjectAllocator::MayLclVarPointToStack(unsigned int lclNum) { assert(m_AnalysisDone); - if (!HasIndex(lclNum)) + if (!IsTrackedLocal(lclNum)) { return false; } - const unsigned bvIndex = LocalToIndex(lclNum); - return BitVecOps::IsMember(&m_bitVecTraits, m_PossiblyStackPointingPointers, bvIndex); + return MayIndexPointToStack(LocalToIndex(lclNum)); +} + +//------------------------------------------------------------------------ +// DoesIndexPointToStack: Returns true iff the resource described by index definitely +// points to a stack-allocated object (or is null) +// +// Arguments: +// index - bv index +// +// Return Value: +// Returns true if so. +// +inline bool ObjectAllocator::DoesIndexPointToStack(unsigned int index) +{ + assert(m_AnalysisDone); + return BitVecOps::IsMember(&m_bitVecTraits, m_DefinitelyStackPointingPointers, index); } //------------------------------------------------------------------------ @@ -314,18 +348,17 @@ inline bool ObjectAllocator::MayLclVarPointToStack(unsigned int lclNum) // Return Value: // Returns true iff local variable definitely points to a stack-allocated object // (or is null) - +// inline bool ObjectAllocator::DoesLclVarPointToStack(unsigned int lclNum) { assert(m_AnalysisDone); - if (!HasIndex(lclNum)) + if (!IsTrackedLocal(lclNum)) { return false; } - const unsigned bvIndex = LocalToIndex(lclNum); - return BitVecOps::IsMember(&m_bitVecTraits, m_DefinitelyStackPointingPointers, bvIndex); + return DoesIndexPointToStack(LocalToIndex(lclNum)); } //===============================================================================