From a75a7da436fe616d8226d8f2a4c5d50c0f447827 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 4 May 2023 23:19:44 -0700 Subject: [PATCH 01/15] Use BitOperations in few hot methods --- src/coreclr/jit/compiler.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index e35d1d0cd666d..395b837d58c86 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -94,7 +94,7 @@ inline RoundLevel getRoundFloatLevel() template inline T genFindLowestBit(T value) { - return (value & (0 - value)); + return BitOperations::BitScanForward(value); } /***************************************************************************** @@ -116,7 +116,7 @@ inline bool genMaxOneBit(T value) template inline bool genExactlyOneBit(T value) { - return ((value != 0) && genMaxOneBit(value)); + return BitOperations::PopCount(value) == 1; } /***************************************************************************** From 46d4d3d88704beca5f0817db3a6e48fc1a14dac6 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 4 May 2023 23:42:22 -0700 Subject: [PATCH 02/15] Do not pass RegisterType for non-arm sarchitectures --- src/coreclr/jit/lsra.cpp | 124 +++++++++++++++++----------------- src/coreclr/jit/lsra.h | 48 ++++++------- src/coreclr/jit/lsrabuild.cpp | 2 +- 3 files changed, 88 insertions(+), 86 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index d803ae8fcaff5..6e294e70890d0 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -309,7 +309,7 @@ regMaskTP LinearScan::getMatchingConstants(regMaskTP mask, Interval* currentInte return result; } -void LinearScan::clearNextIntervalRef(regNumber reg, var_types regType) +void LinearScan::clearNextIntervalRef(regNumber reg ARM_ARG(var_types regType)) { nextIntervalRef[reg] = MaxLocation; #ifdef TARGET_ARM @@ -322,7 +322,7 @@ void LinearScan::clearNextIntervalRef(regNumber reg, var_types regType) #endif } -void LinearScan::clearSpillCost(regNumber reg, var_types regType) +void LinearScan::clearSpillCost(regNumber reg ARM_ARG(var_types regType)) { spillCost[reg] = 0; #ifdef TARGET_ARM @@ -431,7 +431,7 @@ regMaskTP LinearScan::internalFloatRegCandidates() bool LinearScan::isFree(RegRecord* regRecord) { return ((regRecord->assignedInterval == nullptr || !regRecord->assignedInterval->isActive) && - !isRegBusy(regRecord->regNum, regRecord->registerType)); + !isRegBusy(regRecord->regNum ARM_ARG(regRecord->registerType))); } RegRecord* LinearScan::getRegisterRecord(regNumber regNum) @@ -586,7 +586,7 @@ bool LinearScan::conflictingFixedRegReference(regNumber regNum, RefPosition* ref LsraLocation refLocation = refPosition->nodeLocation; RegRecord* regRecord = getRegisterRecord(regNum); - if (isRegInUse(regNum, refPosition->getInterval()->registerType) && + if (isRegInUse(regNum ARM_ARG(refPosition->getInterval()->registerType)) && (regRecord->assignedInterval != refPosition->getInterval())) { return true; @@ -2788,7 +2788,7 @@ bool LinearScan::isMatchingConstant(RegRecord* physRegRecord, RefPosition* refPo return false; } Interval* interval = refPosition->getInterval(); - if (!interval->isConstant || !isRegConstant(physRegRecord->regNum, interval->registerType)) + if (!interval->isConstant || !isRegConstant(physRegRecord->regNum ARM_ARG(interval->registerType))) { return false; } @@ -3130,12 +3130,12 @@ bool LinearScan::isSpillCandidate(Interval* current, RefPosition* refPosition, R LsraLocation refLocation = refPosition->nodeLocation; // We shouldn't be calling this if we haven't already determined that the register is not // busy until the next kill. - assert(!isRegBusy(physRegRecord->regNum, current->registerType)); + assert(!isRegBusy(physRegRecord->regNum ARM_ARG(current->registerType))); // We should already have determined that the register isn't actively in use. #ifdef TARGET_ARM64 - assert(!isRegInUse(physRegRecord->regNum, current->registerType) || refPosition->needsConsecutive); + assert(!isRegInUse(physRegRecord->regNum ARM_ARG(current->registerType)) || refPosition->needsConsecutive); #else - assert(!isRegInUse(physRegRecord->regNum, current->registerType)); + assert(!isRegInUse(physRegRecord->regNum ARM_ARG(current->registerType))); #endif // We shouldn't be calling this if 'refPosition' is a fixed reference to this register. assert(!refPosition->isFixedRefOfRegMask(candidateBit)); @@ -3642,11 +3642,11 @@ void LinearScan::unassignPhysReg(RegRecord* regRec, RefPosition* spillRefPositio else #endif // TARGET_ARM { - clearNextIntervalRef(thisRegNum, assignedInterval->registerType); - clearSpillCost(thisRegNum, assignedInterval->registerType); + clearNextIntervalRef(thisRegNum ARM_ARG(assignedInterval->registerType)); + clearSpillCost(thisRegNum ARM_ARG(assignedInterval->registerType)); checkAndClearInterval(regRec, spillRefPosition); } - makeRegAvailable(regToUnassign, assignedInterval->registerType); + makeRegAvailable(regToUnassign ARM_ARG(assignedInterval->registerType)); RefPosition* nextRefPosition = nullptr; if (spillRefPosition != nullptr) @@ -3744,7 +3744,7 @@ void LinearScan::unassignPhysReg(RegRecord* regRec, RefPosition* spillRefPositio regRec->previousInterval = nullptr; if (regRec->assignedInterval->physReg != thisRegNum) { - clearNextIntervalRef(thisRegNum, regRec->assignedInterval->registerType); + clearNextIntervalRef(thisRegNum ARM_ARG(regRec->assignedInterval->registerType)); } else { @@ -3833,7 +3833,7 @@ void LinearScan::spillGCRefs(RefPosition* killRefPosition) { INDEBUG(killedRegs = true); unassignPhysReg(regRecord, assignedInterval->recentRefPosition); - makeRegAvailable(nextReg, assignedInterval->registerType); + makeRegAvailable(nextReg ARM_ARG(assignedInterval->registerType)); } } INDEBUG(dumpLsraAllocationEvent(killedRegs ? LSRA_EVENT_DONE_KILL_GC_REFS : LSRA_EVENT_NO_GC_KILLS, nullptr, REG_NA, @@ -4145,8 +4145,8 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) { RegRecord* physRegRecord = getRegisterRecord(reg); Interval* assignedInterval = physRegRecord->assignedInterval; - clearNextIntervalRef(reg, physRegRecord->registerType); - clearSpillCost(reg, physRegRecord->registerType); + clearNextIntervalRef(reg ARM_ARG(physRegRecord->registerType)); + clearSpillCost(reg ARM_ARG(physRegRecord->registerType)); if (assignedInterval != nullptr) { assert(assignedInterval->isConstant); @@ -4289,7 +4289,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) assert(targetReg != REG_STK); assert(interval->assignedReg != nullptr && interval->assignedReg->regNum == targetReg && interval->assignedReg->assignedInterval == interval); - liveRegs |= getRegMask(targetReg, interval->registerType); + liveRegs |= getRegMask(targetReg ARM_ARG(interval->registerType)); continue; } } @@ -4319,7 +4319,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) // likely to match other assignments this way. targetReg = interval->physReg; interval->isActive = true; - liveRegs |= getRegMask(targetReg, interval->registerType); + liveRegs |= getRegMask(targetReg ARM_ARG(interval->registerType)); INDEBUG(inactiveRegs |= genRegMask(targetReg)); setVarReg(inVarToRegMap, varIndex, targetReg); } @@ -4331,7 +4331,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) if (targetReg != REG_STK) { RegRecord* targetRegRecord = getRegisterRecord(targetReg); - liveRegs |= getRegMask(targetReg, interval->registerType); + liveRegs |= getRegMask(targetReg ARM_ARG(interval->registerType)); if (!allocationPassComplete) { updateNextIntervalRef(targetReg, interval); @@ -4394,7 +4394,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) RegRecord* physRegRecord = getRegisterRecord(reg); if ((liveRegs & genRegMask(reg)) == 0) { - makeRegAvailable(reg, physRegRecord->registerType); + makeRegAvailable(reg ARM_ARG(physRegRecord->registerType)); Interval* assignedInterval = physRegRecord->assignedInterval; if (assignedInterval != nullptr) @@ -4541,7 +4541,7 @@ void LinearScan::makeRegisterInactive(RegRecord* physRegRecord) assignedInterval->isActive = false; if (assignedInterval->isConstant) { - clearNextIntervalRef(physRegRecord->regNum, assignedInterval->registerType); + clearNextIntervalRef(physRegRecord->regNum ARM_ARG(assignedInterval->registerType)); } } } @@ -4570,8 +4570,8 @@ void LinearScan::makeRegisterInactive(RegRecord* physRegRecord) void LinearScan::freeRegister(RegRecord* physRegRecord) { Interval* assignedInterval = physRegRecord->assignedInterval; - makeRegAvailable(physRegRecord->regNum, physRegRecord->registerType); - clearSpillCost(physRegRecord->regNum, physRegRecord->registerType); + makeRegAvailable(physRegRecord->regNum ARM_ARG(physRegRecord->registerType)); + clearSpillCost(physRegRecord->regNum ARM_ARG(physRegRecord->registerType)); makeRegisterInactive(physRegRecord); if (assignedInterval != nullptr) @@ -4691,14 +4691,14 @@ void LinearScan::allocateRegisters() { updateNextIntervalRef(reg, interval); updateSpillCost(reg, interval); - setRegInUse(reg, interval->registerType); - INDEBUG(registersToDump |= getRegMask(reg, interval->registerType)); + setRegInUse(reg ARM_ARG(interval->registerType)); + INDEBUG(registersToDump |= getRegMask(reg ARM_ARG(interval->registerType))); } } else { - clearNextIntervalRef(reg, physRegRecord->registerType); - clearSpillCost(reg, physRegRecord->registerType); + clearNextIntervalRef(reg ARM_ARG(physRegRecord->registerType)); + clearSpillCost(reg ARM_ARG(physRegRecord->registerType)); } } @@ -4751,7 +4751,7 @@ void LinearScan::allocateRegisters() tempRegsToMakeInactive &= ~nextRegBit; regNumber nextReg = genRegNumFromMask(nextRegBit); RegRecord* regRecord = getRegisterRecord(nextReg); - clearSpillCost(regRecord->regNum, regRecord->registerType); + clearSpillCost(regRecord->regNum ARM_ARG(regRecord->registerType)); makeRegisterInactive(regRecord); } if (currentRefPosition.nodeLocation > prevLocation) @@ -4873,7 +4873,8 @@ void LinearScan::allocateRegisters() { continue; } - assert(assignedInterval->isConstant == isRegConstant(reg, assignedInterval->registerType)); + assert(assignedInterval->isConstant == + isRegConstant(reg ARM_ARG(assignedInterval->registerType))); if (assignedInterval->isActive) { // If this is not the register most recently allocated, it must be from a copyReg, @@ -4901,14 +4902,14 @@ void LinearScan::allocateRegisters() if (isAssignedReg) { assert(nextIntervalRef[reg] == assignedInterval->getNextRefLocation()); - assert(!isRegAvailable(reg, assignedInterval->registerType)); + assert(!isRegAvailable(reg ARM_ARG(assignedInterval->registerType))); assert((recentRefPosition == nullptr) || (spillCost[reg] == getSpillWeight(physRegRecord))); } else { assert((nextIntervalRef[reg] == MaxLocation) || - isRegBusy(reg, assignedInterval->registerType)); + isRegBusy(reg ARM_ARG(assignedInterval->registerType))); } } else @@ -4920,7 +4921,7 @@ void LinearScan::allocateRegisters() else { assert(nextIntervalRef[reg] == MaxLocation); - assert(isRegAvailable(reg, assignedInterval->registerType)); + assert(isRegAvailable(reg ARM_ARG(assignedInterval->registerType))); assert(spillCost[reg] == 0); } } @@ -4929,8 +4930,8 @@ void LinearScan::allocateRegisters() else { // Available registers should not hold constants - assert(isRegAvailable(reg, physRegRecord->registerType)); - assert(!isRegConstant(reg, physRegRecord->registerType)); + assert(isRegAvailable(reg ARM_ARG(physRegRecord->registerType))); + assert(!isRegConstant(reg ARM_ARG(physRegRecord->registerType))); assert(nextIntervalRef[reg] == MaxLocation); assert(spillCost[reg] == 0); } @@ -5043,7 +5044,7 @@ void LinearScan::allocateRegisters() { if (assignedInterval != nullptr && !assignedInterval->isActive && assignedInterval->isConstant) { - clearConstantReg(regRecord->regNum, assignedInterval->registerType); + clearConstantReg(regRecord->regNum ARM_ARG(assignedInterval->registerType)); regRecord->assignedInterval = nullptr; spillCost[regRecord->regNum] = 0; @@ -5067,8 +5068,8 @@ void LinearScan::allocateRegisters() if (assignedInterval != nullptr) { unassignPhysReg(regRecord, assignedInterval->recentRefPosition); - clearConstantReg(regRecord->regNum, assignedInterval->registerType); - makeRegAvailable(regRecord->regNum, assignedInterval->registerType); + clearConstantReg(regRecord->regNum ARM_ARG(assignedInterval->registerType)); + makeRegAvailable(regRecord->regNum ARM_ARG(assignedInterval->registerType)); } clearRegBusyUntilKill(regRecord->regNum); INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_KEPT_ALLOCATION, nullptr, regRecord->regNum)); @@ -5149,9 +5150,9 @@ void LinearScan::allocateRegisters() setIntervalAsSpilled(currentInterval); if (assignedRegister != REG_NA) { - clearNextIntervalRef(assignedRegister, currentInterval->registerType); - clearSpillCost(assignedRegister, currentInterval->registerType); - makeRegAvailable(assignedRegister, currentInterval->registerType); + clearNextIntervalRef(assignedRegister ARM_ARG(currentInterval->registerType)); + clearSpillCost(assignedRegister ARM_ARG(currentInterval->registerType)); + makeRegAvailable(assignedRegister ARM_ARG(currentInterval->registerType)); } } } @@ -5208,7 +5209,7 @@ void LinearScan::allocateRegisters() unassignPhysReg(regRecord, currentInterval->firstRefPosition); if (assignedInterval->isConstant) { - clearConstantReg(assignedRegister, assignedInterval->registerType); + clearConstantReg(assignedRegister ARM_ARG(assignedInterval->registerType)); } INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_NO_REG_ALLOCATED, currentInterval)); } @@ -5266,7 +5267,7 @@ void LinearScan::allocateRegisters() // special putarg_reg doesn't get spilled and re-allocated prior to // its use at the call node. This is ensured by marking physical reg // record as busy until next kill. - setRegBusyUntilKill(srcInterval->physReg, srcInterval->registerType); + setRegBusyUntilKill(srcInterval->physReg ARM_ARG(srcInterval->registerType)); } else { @@ -5308,7 +5309,7 @@ void LinearScan::allocateRegisters() else { currentInterval->isActive = true; - setRegInUse(assignedRegister, currentInterval->registerType); + setRegInUse(assignedRegister ARM_ARG(currentInterval->registerType)); updateSpillCost(assignedRegister, currentInterval); } updateNextIntervalRef(assignedRegister, currentInterval); @@ -5402,7 +5403,7 @@ void LinearScan::allocateRegisters() RegRecord* physRegRecord = getRegisterRecord(assignedRegister); assert((assignedRegBit == currentRefPosition.registerAssignment) || (physRegRecord->assignedInterval == currentInterval) || - !isRegInUse(assignedRegister, currentInterval->registerType)); + !isRegInUse(assignedRegister ARM_ARG(currentInterval->registerType))); if (conflictingFixedRegReference(assignedRegister, ¤tRefPosition)) { // We may have already reassigned the register to the conflicting reference. @@ -5410,7 +5411,7 @@ void LinearScan::allocateRegisters() if (physRegRecord->assignedInterval == currentInterval) { unassignPhysRegNoSpill(physRegRecord); - clearConstantReg(assignedRegister, currentInterval->registerType); + clearConstantReg(assignedRegister ARM_ARG(currentInterval->registerType)); } currentRefPosition.moveReg = true; assignedRegister = REG_NA; @@ -5443,9 +5444,10 @@ void LinearScan::allocateRegisters() if (copyReg != assignedRegister) { - lastAllocatedRefPosition = ¤tRefPosition; - regMaskTP copyRegMask = getRegMask(copyReg, currentInterval->registerType); - regMaskTP assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); + lastAllocatedRefPosition = ¤tRefPosition; + regMaskTP copyRegMask = getRegMask(copyReg ARM_ARG(currentInterval->registerType)); + regMaskTP assignedRegMask = + getRegMask(assignedRegister ARM_ARG(currentInterval->registerType)); if ((consecutiveRegsInUseThisLocation & assignedRegMask) != RBM_NONE) { @@ -5480,8 +5482,8 @@ void LinearScan::allocateRegisters() currentRefPosition.moveReg = true; currentRefPosition.copyReg = false; } - clearNextIntervalRef(copyReg, currentInterval->registerType); - clearSpillCost(copyReg, currentInterval->registerType); + clearNextIntervalRef(copyReg ARM_ARG(currentInterval->registerType)); + clearSpillCost(copyReg ARM_ARG(currentInterval->registerType)); updateNextIntervalRef(assignedRegister, currentInterval); updateSpillCost(assignedRegister, currentInterval); } @@ -5541,8 +5543,8 @@ void LinearScan::allocateRegisters() } lastAllocatedRefPosition = ¤tRefPosition; - regMaskTP copyRegMask = getRegMask(copyReg, currentInterval->registerType); - regMaskTP assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); + regMaskTP copyRegMask = getRegMask(copyReg ARM_ARG(currentInterval->registerType)); + regMaskTP assignedRegMask = getRegMask(assignedRegister ARM_ARG(currentInterval->registerType)); #ifdef TARGET_ARM64 if (hasConsecutiveRegister && currentRefPosition.needsConsecutive) @@ -5587,8 +5589,8 @@ void LinearScan::allocateRegisters() currentRefPosition.moveReg = true; currentRefPosition.copyReg = false; } - clearNextIntervalRef(copyReg, currentInterval->registerType); - clearSpillCost(copyReg, currentInterval->registerType); + clearNextIntervalRef(copyReg ARM_ARG(currentInterval->registerType)); + clearSpillCost(copyReg ARM_ARG(currentInterval->registerType)); updateNextIntervalRef(assignedRegister, currentInterval); updateSpillCost(assignedRegister, currentInterval); continue; @@ -5596,7 +5598,7 @@ void LinearScan::allocateRegisters() else { INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_NEEDS_NEW_REG, nullptr, assignedRegister)); - regsToFree |= getRegMask(assignedRegister, currentInterval->registerType); + regsToFree |= getRegMask(assignedRegister ARM_ARG(currentInterval->registerType)); // We want a new register, but we don't want this to be considered a spill. assignedRegister = REG_NA; if (physRegRecord->assignedInterval == currentInterval) @@ -5764,7 +5766,7 @@ void LinearScan::allocateRegisters() if (assignedRegister != REG_NA) { assignedRegBit = genRegMask(assignedRegister); - regMaskTP regMask = getRegMask(assignedRegister, currentInterval->registerType); + regMaskTP regMask = getRegMask(assignedRegister ARM_ARG(currentInterval->registerType)); regsInUseThisLocation |= regMask; if (currentRefPosition.delayRegFree) { @@ -6012,22 +6014,22 @@ void LinearScan::updateAssignedInterval(RegRecord* reg, Interval* interval, Regi reg->assignedInterval = interval; if (interval != nullptr) { - setRegInUse(reg->regNum, interval->registerType); + setRegInUse(reg->regNum ARM_ARG(interval->registerType)); if (interval->isConstant) { - setConstantReg(reg->regNum, interval->registerType); + setConstantReg(reg->regNum ARM_ARG(interval->registerType)); } else { - clearConstantReg(reg->regNum, interval->registerType); + clearConstantReg(reg->regNum ARM_ARG(interval->registerType)); } updateNextIntervalRef(reg->regNum, interval); updateSpillCost(reg->regNum, interval); } else { - clearNextIntervalRef(reg->regNum, reg->registerType); - clearSpillCost(reg->regNum, reg->registerType); + clearNextIntervalRef(reg->regNum ARM_ARG(reg->registerType)); + clearSpillCost(reg->regNum ARM_ARG(reg->registerType)); } } @@ -10076,7 +10078,7 @@ void LinearScan::dumpLsraAllocationEvent( } if ((interval != nullptr) && (reg != REG_NA) && (reg != REG_STK)) { - registersToDump |= getRegMask(reg, interval->registerType); + registersToDump |= getRegMask(reg ARM_ARG(interval->registerType)); dumpRegRecordTitleIfNeeded(); } diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index d869f8f8a126c..b72627a65d978 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1723,7 +1723,7 @@ class LinearScan : public LinearScanInterface //----------------------------------------------------------------------- regMaskTP m_AvailableRegs; - regNumber getRegForType(regNumber reg, var_types regType) + regNumber getRegForType(regNumber reg ARM_ARG(var_types regType)) { #ifdef TARGET_ARM if ((regType == TYP_DOUBLE) && !genIsValidDoubleReg(reg)) @@ -1734,9 +1734,9 @@ class LinearScan : public LinearScanInterface return reg; } - regMaskTP getRegMask(regNumber reg, var_types regType) + regMaskTP getRegMask(regNumber reg ARM_ARG(var_types regType)) { - reg = getRegForType(reg, regType); + reg = getRegForType(reg ARM_ARG(regType)); regMaskTP regMask = genRegMask(reg); #ifdef TARGET_ARM if (regType == TYP_DOUBLE) @@ -1754,34 +1754,34 @@ class LinearScan : public LinearScanInterface m_RegistersWithConstants = RBM_NONE; } - bool isRegAvailable(regNumber reg, var_types regType) + bool isRegAvailable(regNumber reg ARM_ARG(var_types regType)) { - regMaskTP regMask = getRegMask(reg, regType); + regMaskTP regMask = getRegMask(reg ARM_ARG(regType)); return (m_AvailableRegs & regMask) == regMask; } void setRegsInUse(regMaskTP regMask) { m_AvailableRegs &= ~regMask; } - void setRegInUse(regNumber reg, var_types regType) + void setRegInUse(regNumber reg ARM_ARG(var_types regType)) { - regMaskTP regMask = getRegMask(reg, regType); + regMaskTP regMask = getRegMask(reg ARM_ARG(regType)); setRegsInUse(regMask); } void makeRegsAvailable(regMaskTP regMask) { m_AvailableRegs |= regMask; } - void makeRegAvailable(regNumber reg, var_types regType) + void makeRegAvailable(regNumber reg ARM_ARG(var_types regType)) { - regMaskTP regMask = getRegMask(reg, regType); + regMaskTP regMask = getRegMask(reg ARM_ARG(regType)); makeRegsAvailable(regMask); } - void clearNextIntervalRef(regNumber reg, var_types regType); + void clearNextIntervalRef(regNumber reg ARM_ARG(var_types regType)); void updateNextIntervalRef(regNumber reg, Interval* interval); - void clearSpillCost(regNumber reg, var_types regType); + void clearSpillCost(regNumber reg ARM_ARG(var_types regType)); void updateSpillCost(regNumber reg, Interval* interval); FORCEINLINE void updateRegsFreeBusyState(RefPosition& refPosition, @@ -1791,18 +1791,18 @@ class LinearScan : public LinearScanInterface DEBUG_ARG(regNumber assignedReg)); regMaskTP m_RegistersWithConstants; - void clearConstantReg(regNumber reg, var_types regType) + void clearConstantReg(regNumber reg ARM_ARG(var_types regType)) { - m_RegistersWithConstants &= ~getRegMask(reg, regType); + m_RegistersWithConstants &= ~getRegMask(reg ARM_ARG(regType)); } - void setConstantReg(regNumber reg, var_types regType) + void setConstantReg(regNumber reg ARM_ARG(var_types regType)) { - m_RegistersWithConstants |= getRegMask(reg, regType); + m_RegistersWithConstants |= getRegMask(reg ARM_ARG(regType)); } - bool isRegConstant(regNumber reg, var_types regType) + bool isRegConstant(regNumber reg ARM_ARG(var_types regType)) { - reg = getRegForType(reg, regType); - regMaskTP regMask = getRegMask(reg, regType); + reg = getRegForType(reg ARM_ARG(regType)); + regMaskTP regMask = getRegMask(reg ARM_ARG(regType)); return (m_RegistersWithConstants & regMask) == regMask; } regMaskTP getMatchingConstants(regMaskTP mask, Interval* currentInterval, RefPosition* refPosition); @@ -1842,23 +1842,23 @@ class LinearScan : public LinearScanInterface #ifdef TARGET_ARM64 regMaskTP consecutiveRegsInUseThisLocation; #endif - bool isRegBusy(regNumber reg, var_types regType) + bool isRegBusy(regNumber reg ARM_ARG(var_types regType)) { - regMaskTP regMask = getRegMask(reg, regType); + regMaskTP regMask = getRegMask(reg ARM_ARG(regType)); return (regsBusyUntilKill & regMask) != RBM_NONE; } - void setRegBusyUntilKill(regNumber reg, var_types regType) + void setRegBusyUntilKill(regNumber reg ARM_ARG(var_types regType)) { - regsBusyUntilKill |= getRegMask(reg, regType); + regsBusyUntilKill |= getRegMask(reg ARM_ARG(regType)); } void clearRegBusyUntilKill(regNumber reg) { regsBusyUntilKill &= ~genRegMask(reg); } - bool isRegInUse(regNumber reg, var_types regType) + bool isRegInUse(regNumber reg ARM_ARG(var_types regType)) { - regMaskTP regMask = getRegMask(reg, regType); + regMaskTP regMask = getRegMask(reg ARM_ARG(regType)); return (regsInUseThisLocation & regMask) != RBM_NONE; } diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 0f11acdadb701..4ebee0b810f33 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -2273,7 +2273,7 @@ void LinearScan::buildIntervals() assert(inArgReg < REG_COUNT); mask = genRegMask(inArgReg); assignPhysReg(inArgReg, interval); - INDEBUG(registersToDump |= getRegMask(inArgReg, interval->registerType)); + INDEBUG(registersToDump |= getRegMask(inArgReg ARM_ARG(interval->registerType))); } RefPosition* pos = newRefPosition(interval, MinLocation, RefTypeParamDef, nullptr, mask); pos->setRegOptional(true); From ade6707c1d94405bc61bf1ec74740b6fc56d4397 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 5 May 2023 08:30:09 -0700 Subject: [PATCH 03/15] Add clearAssignedInterval() --- src/coreclr/jit/lsra.cpp | 38 ++++++++++++++++++++++++++++++++++++++ src/coreclr/jit/lsra.h | 1 + 2 files changed, 39 insertions(+) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 6e294e70890d0..8ed24b6693e44 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -5971,6 +5971,44 @@ void LinearScan::allocateRegisters() #endif // DEBUG } +//----------------------------------------------------------------------------- +// clearAssignedInterval: Clear assigned interval of register. +// +// Arguments: +// reg - register to be updated +// regType - register type +// +// Return Value: +// None +// +// Note: +// For ARM32, two float registers consisting a double register are cleared +// together when "regType" is TYP_DOUBLE. +// +void LinearScan::clearAssignedInterval(RegRecord* reg ARM_ARG(RegisterType regType)) +{ +#ifdef TARGET_ARM + if (regType == TYP_DOUBLE) + { + RegRecord* anotherHalfReg = findAnotherHalfRegRec(reg); + regNumbre doubleReg = genIsValidDoubleReg(reg->regNum) ? reg->regNum : anotherHalfReg->regNum; + + reg->assignedInterval = nullptr; + anotherHalfReg->assignedInterval = interval; + + clearNextIntervalRef(doubleReg, TYP_DOUBLE); + clearSpillCost(doubleReg, TYP_DOUBLE); + clearConstantReg(doubleReg, TYP_DOUBLE); + + return; + } +#endif // TARGET_ARM + + reg->assignedInterval = nullptr; + clearNextIntervalRef(reg->regNum ARM_ARG(reg->registerType)); + clearSpillCost(reg->regNum ARM_ARG(reg->registerType)); +} + //----------------------------------------------------------------------------- // updateAssignedInterval: Update assigned interval of register. // diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index b72627a65d978..a466d778850a6 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1016,6 +1016,7 @@ class LinearScan : public LinearScanInterface bool canSpillDoubleReg(RegRecord* physRegRecord, LsraLocation refLocation); void unassignDoublePhysReg(RegRecord* doubleRegRecord); #endif + void clearAssignedInterval(RegRecord* reg ARM_ARG(RegisterType regType)); void updateAssignedInterval(RegRecord* reg, Interval* interval, RegisterType regType); void updatePreviousInterval(RegRecord* reg, Interval* interval, RegisterType regType); bool canRestorePreviousInterval(RegRecord* regRec, Interval* assignedInterval); From b9b392016f9451f0684a9d4a46f27520330afd25 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 5 May 2023 13:52:31 -0700 Subject: [PATCH 04/15] Consume clearAssignedInterval() --- src/coreclr/jit/lsra.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 8ed24b6693e44..2dfee9b23f2c9 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -3521,7 +3521,7 @@ void LinearScan::checkAndClearInterval(RegRecord* regRec, RefPosition* spillRefP assert(spillRefPosition->getInterval() == assignedInterval); } - updateAssignedInterval(regRec, nullptr, assignedInterval->registerType); + clearAssignedInterval(regRec ARM_ARG(assignedInterval->registerType)); } //------------------------------------------------------------------------ @@ -3780,7 +3780,7 @@ void LinearScan::unassignPhysReg(RegRecord* regRec, RefPosition* spillRefPositio } else { - updateAssignedInterval(regRec, nullptr, assignedInterval->registerType); + clearAssignedInterval(regRec ARM_ARG(assignedInterval->registerType)); updatePreviousInterval(regRec, nullptr, assignedInterval->registerType); } } @@ -4110,7 +4110,7 @@ void LinearScan::unassignIntervalBlockStart(RegRecord* regRecord, VarToRegMap in else { // This interval is no longer assigned to this register. - updateAssignedInterval(regRecord, nullptr, assignedInterval->registerType); + clearAssignedInterval(regRecord ARM_ARG(assignedInterval->registerType)); } } } @@ -4418,7 +4418,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) { // This interval may still be active, but was in another register in an // intervening block. - updateAssignedInterval(physRegRecord, nullptr, assignedInterval->registerType); + clearAssignedInterval(physRegRecord ARM_ARG(assignedInterval->registerType)); } #ifdef TARGET_ARM @@ -5991,10 +5991,10 @@ void LinearScan::clearAssignedInterval(RegRecord* reg ARM_ARG(RegisterType regTy if (regType == TYP_DOUBLE) { RegRecord* anotherHalfReg = findAnotherHalfRegRec(reg); - regNumbre doubleReg = genIsValidDoubleReg(reg->regNum) ? reg->regNum : anotherHalfReg->regNum; + regNumber doubleReg = genIsValidDoubleReg(reg->regNum) ? reg->regNum : anotherHalfReg->regNum; - reg->assignedInterval = nullptr; - anotherHalfReg->assignedInterval = interval; + reg->assignedInterval = nullptr; + anotherHalfReg->assignedInterval = nullptr; clearNextIntervalRef(doubleReg, TYP_DOUBLE); clearSpillCost(doubleReg, TYP_DOUBLE); @@ -6222,7 +6222,7 @@ void LinearScan::resolveLocalRef(BasicBlock* block, GenTreeLclVar* treeNode, Ref varDsc->SetRegNum(REG_STK); if (interval->assignedReg != nullptr && interval->assignedReg->assignedInterval == interval) { - updateAssignedInterval(interval->assignedReg, nullptr, interval->registerType); + clearAssignedInterval(interval->assignedReg ARM_ARG(interval->registerType)); } interval->assignedReg = nullptr; interval->physReg = REG_NA; @@ -6255,7 +6255,7 @@ void LinearScan::resolveLocalRef(BasicBlock* block, GenTreeLclVar* treeNode, Ref RegRecord* oldRegRecord = getRegisterRecord(oldAssignedReg); if (oldRegRecord->assignedInterval == interval) { - updateAssignedInterval(oldRegRecord, nullptr, interval->registerType); + clearAssignedInterval(oldRegRecord ARM_ARG(interval->registerType)); } } } @@ -6467,7 +6467,7 @@ void LinearScan::resolveLocalRef(BasicBlock* block, GenTreeLclVar* treeNode, Ref interval->assignedReg = nullptr; interval->physReg = REG_NA; - updateAssignedInterval(physRegRecord, nullptr, interval->registerType); + clearAssignedInterval(physRegRecord ARM_ARG(interval->registerType)); } else { From 5ff83da75cb6fc6e142b5b51236058a17e221eca Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 5 May 2023 14:02:28 -0700 Subject: [PATCH 05/15] Do not pass RegisterType for updateInterval() --- src/coreclr/jit/lsra.cpp | 39 +++++++++++++++++---------------------- src/coreclr/jit/lsra.h | 6 +++--- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 2dfee9b23f2c9..fd5482e7d3987 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -2945,7 +2945,8 @@ regNumber LinearScan::allocateReg(Interval* currentInterval, } else if (wasAssigned) { - updatePreviousInterval(availablePhysRegRecord, assignedInterval, assignedInterval->registerType); + updatePreviousInterval(availablePhysRegRecord, + assignedInterval ARM_ARG(assignedInterval->registerType)); } else { @@ -3284,7 +3285,7 @@ void LinearScan::checkAndAssignInterval(RegRecord* regRec, Interval* interval) } #endif - updateAssignedInterval(regRec, interval, interval->registerType); + updateAssignedInterval(regRec, interval ARM_ARG(interval->registerType)); } // Assign the given physical register interval to the given interval @@ -3781,7 +3782,7 @@ void LinearScan::unassignPhysReg(RegRecord* regRec, RefPosition* spillRefPositio else { clearAssignedInterval(regRec ARM_ARG(assignedInterval->registerType)); - updatePreviousInterval(regRec, nullptr, assignedInterval->registerType); + updatePreviousInterval(regRec, nullptr ARM_ARG(assignedInterval->registerType)); } } @@ -5990,8 +5991,8 @@ void LinearScan::clearAssignedInterval(RegRecord* reg ARM_ARG(RegisterType regTy #ifdef TARGET_ARM if (regType == TYP_DOUBLE) { - RegRecord* anotherHalfReg = findAnotherHalfRegRec(reg); - regNumber doubleReg = genIsValidDoubleReg(reg->regNum) ? reg->regNum : anotherHalfReg->regNum; + RegRecord* anotherHalfReg = findAnotherHalfRegRec(reg); + regNumber doubleReg = genIsValidDoubleReg(reg->regNum) ? reg->regNum : anotherHalfReg->regNum; reg->assignedInterval = nullptr; anotherHalfReg->assignedInterval = nullptr; @@ -6024,8 +6025,10 @@ void LinearScan::clearAssignedInterval(RegRecord* reg ARM_ARG(RegisterType regTy // For ARM32, two float registers consisting a double register are updated // together when "regType" is TYP_DOUBLE. // -void LinearScan::updateAssignedInterval(RegRecord* reg, Interval* interval, RegisterType regType) +void LinearScan::updateAssignedInterval(RegRecord* reg, Interval* interval ARM_ARG(RegisterType regType)) { + assert(interval != nullptr); + #ifdef TARGET_ARM // Update overlapping floating point register for TYP_DOUBLE. Interval* oldAssignedInterval = reg->assignedInterval; @@ -6050,25 +6053,17 @@ void LinearScan::updateAssignedInterval(RegRecord* reg, Interval* interval, Regi } #endif reg->assignedInterval = interval; - if (interval != nullptr) + setRegInUse(reg->regNum ARM_ARG(interval->registerType)); + if (interval->isConstant) { - setRegInUse(reg->regNum ARM_ARG(interval->registerType)); - if (interval->isConstant) - { - setConstantReg(reg->regNum ARM_ARG(interval->registerType)); - } - else - { - clearConstantReg(reg->regNum ARM_ARG(interval->registerType)); - } - updateNextIntervalRef(reg->regNum, interval); - updateSpillCost(reg->regNum, interval); + setConstantReg(reg->regNum ARM_ARG(interval->registerType)); } else { - clearNextIntervalRef(reg->regNum ARM_ARG(reg->registerType)); - clearSpillCost(reg->regNum ARM_ARG(reg->registerType)); + clearConstantReg(reg->regNum ARM_ARG(interval->registerType)); } + updateNextIntervalRef(reg->regNum, interval); + updateSpillCost(reg->regNum, interval); } //----------------------------------------------------------------------------- @@ -6090,7 +6085,7 @@ void LinearScan::updateAssignedInterval(RegRecord* reg, Interval* interval, Regi // For ARM32, two float registers consisting a double register are updated // together when "regType" is TYP_DOUBLE. // -void LinearScan::updatePreviousInterval(RegRecord* reg, Interval* interval, RegisterType regType) +void LinearScan::updatePreviousInterval(RegRecord* reg, Interval* interval ARM_ARG(RegisterType regType)) { reg->previousInterval = interval; @@ -6474,7 +6469,7 @@ void LinearScan::resolveLocalRef(BasicBlock* block, GenTreeLclVar* treeNode, Ref interval->isActive = true; interval->assignedReg = physRegRecord; - updateAssignedInterval(physRegRecord, interval, interval->registerType); + updateAssignedInterval(physRegRecord, interval ARM_ARG(interval->registerType)); } } diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index a466d778850a6..26fb73c3dacc8 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1016,9 +1016,9 @@ class LinearScan : public LinearScanInterface bool canSpillDoubleReg(RegRecord* physRegRecord, LsraLocation refLocation); void unassignDoublePhysReg(RegRecord* doubleRegRecord); #endif - void clearAssignedInterval(RegRecord* reg ARM_ARG(RegisterType regType)); - void updateAssignedInterval(RegRecord* reg, Interval* interval, RegisterType regType); - void updatePreviousInterval(RegRecord* reg, Interval* interval, RegisterType regType); + void clearAssignedInterval(RegRecord* reg ARM_ARG(RegisterType regType)); + void updateAssignedInterval(RegRecord* reg, Interval* interval ARM_ARG(RegisterType regType)); + void updatePreviousInterval(RegRecord* reg, Interval* interval ARM_ARG(RegisterType regType)); bool canRestorePreviousInterval(RegRecord* regRec, Interval* assignedInterval); bool isAssignedToInterval(Interval* interval, RegRecord* regRec); bool isRefPositionActive(RefPosition* refPosition, LsraLocation refLocation); From 83436fe903b10cb45d785e9486fb46adc5fd814a Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 5 May 2023 14:34:17 -0700 Subject: [PATCH 06/15] Revert the change in genFindLowestBit() --- src/coreclr/jit/compiler.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 395b837d58c86..cec779f657ea8 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -94,7 +94,7 @@ inline RoundLevel getRoundFloatLevel() template inline T genFindLowestBit(T value) { - return BitOperations::BitScanForward(value); + return (value & (0 - value)); } /***************************************************************************** From d76f001ff2feb32bbe315a807301df54eaed72c6 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 5 May 2023 21:21:40 -0700 Subject: [PATCH 07/15] Revert "Use BitOperations in few hot methods" This reverts commit a75a7da436fe616d8226d8f2a4c5d50c0f447827. --- src/coreclr/jit/compiler.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index cec779f657ea8..e35d1d0cd666d 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -116,7 +116,7 @@ inline bool genMaxOneBit(T value) template inline bool genExactlyOneBit(T value) { - return BitOperations::PopCount(value) == 1; + return ((value != 0) && genMaxOneBit(value)); } /***************************************************************************** From 20e497c095d3c155c35049e2fdacfcad275b9a2d Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 8 May 2023 09:15:39 -0700 Subject: [PATCH 08/15] Add the missing case for clearAssignedInterval() --- src/coreclr/jit/lsra.cpp | 18 +++++++++++++----- src/coreclr/jit/lsra.h | 2 +- src/coreclr/scripts/superpmi.py | 2 +- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index fd5482e7d3987..9ebc077f8d96d 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -686,6 +686,7 @@ LinearScan::LinearScan(Compiler* theCompiler) , refPositions(theCompiler->getAllocator(CMK_LSRA_RefPosition)) , listNodePool(theCompiler) { + counter = 0; #if defined(TARGET_XARCH) availableRegCount = ACTUAL_REG_COUNT; @@ -5989,19 +5990,26 @@ void LinearScan::allocateRegisters() void LinearScan::clearAssignedInterval(RegRecord* reg ARM_ARG(RegisterType regType)) { #ifdef TARGET_ARM + regNumber doubleReg = REG_NA; + Interval* oldAssignedInterval = reg->assignedInterval; if (regType == TYP_DOUBLE) { RegRecord* anotherHalfReg = findAnotherHalfRegRec(reg); - regNumber doubleReg = genIsValidDoubleReg(reg->regNum) ? reg->regNum : anotherHalfReg->regNum; - - reg->assignedInterval = nullptr; + doubleReg = genIsValidDoubleReg(reg->regNum) ? reg->regNum : anotherHalfReg->regNum; anotherHalfReg->assignedInterval = nullptr; + } + else if ((oldAssignedInterval != nullptr) && (oldAssignedInterval->registerType == TYP_DOUBLE)) + { + RegRecord* anotherHalfReg = findAnotherHalfRegRec(reg); + doubleReg = genIsValidDoubleReg(reg->regNum) ? reg->regNum : anotherHalfReg->regNum; + anotherHalfReg->assignedInterval = nullptr; + } + if (doubleReg != REG_NA) + { clearNextIntervalRef(doubleReg, TYP_DOUBLE); clearSpillCost(doubleReg, TYP_DOUBLE); clearConstantReg(doubleReg, TYP_DOUBLE); - - return; } #endif // TARGET_ARM diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 26fb73c3dacc8..58fa9ed1ef00b 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1553,7 +1553,7 @@ class LinearScan : public LinearScanInterface private: Compiler* compiler; - + int counter; CompAllocator getAllocator(Compiler* comp) { return comp->getAllocator(CMK_LSRA); diff --git a/src/coreclr/scripts/superpmi.py b/src/coreclr/scripts/superpmi.py index 4fb1c207b064b..5dde1857d8823 100644 --- a/src/coreclr/scripts/superpmi.py +++ b/src/coreclr/scripts/superpmi.py @@ -1347,7 +1347,7 @@ def replay(self): common_flags = [ "-v", "ewi", # display errors, warnings, missing, jit info - "-r", os.path.join(temp_location, "repro") # Repro name, create .mc repro files + #"-r", os.path.join(temp_location, "repro") # Repro name, create .mc repro files ] if self.coreclr_args.altjit: From 9fda67242e0b31eef1d7059642e6cce67158a537 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 8 May 2023 10:36:14 -0700 Subject: [PATCH 09/15] Remove logging --- src/coreclr/jit/lsra.cpp | 1 - src/coreclr/jit/lsra.h | 1 - 2 files changed, 2 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 9ebc077f8d96d..8904b6bcc2525 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -686,7 +686,6 @@ LinearScan::LinearScan(Compiler* theCompiler) , refPositions(theCompiler->getAllocator(CMK_LSRA_RefPosition)) , listNodePool(theCompiler) { - counter = 0; #if defined(TARGET_XARCH) availableRegCount = ACTUAL_REG_COUNT; diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 58fa9ed1ef00b..2e58e2401fd36 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1553,7 +1553,6 @@ class LinearScan : public LinearScanInterface private: Compiler* compiler; - int counter; CompAllocator getAllocator(Compiler* comp) { return comp->getAllocator(CMK_LSRA); From 40e49f9a49ca4d82579cfe7ce8df5242330122d4 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 8 May 2023 10:41:25 -0700 Subject: [PATCH 10/15] jit formatting --- src/coreclr/jit/lsra.cpp | 6 +++--- src/coreclr/jit/lsra.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 8904b6bcc2525..f72a22bb1243e 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -5989,12 +5989,12 @@ void LinearScan::allocateRegisters() void LinearScan::clearAssignedInterval(RegRecord* reg ARM_ARG(RegisterType regType)) { #ifdef TARGET_ARM - regNumber doubleReg = REG_NA; + regNumber doubleReg = REG_NA; Interval* oldAssignedInterval = reg->assignedInterval; if (regType == TYP_DOUBLE) { - RegRecord* anotherHalfReg = findAnotherHalfRegRec(reg); - doubleReg = genIsValidDoubleReg(reg->regNum) ? reg->regNum : anotherHalfReg->regNum; + RegRecord* anotherHalfReg = findAnotherHalfRegRec(reg); + doubleReg = genIsValidDoubleReg(reg->regNum) ? reg->regNum : anotherHalfReg->regNum; anotherHalfReg->assignedInterval = nullptr; } else if ((oldAssignedInterval != nullptr) && (oldAssignedInterval->registerType == TYP_DOUBLE)) diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 2e58e2401fd36..43a6009d9db35 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1552,7 +1552,7 @@ class LinearScan : public LinearScanInterface #endif // !TRACK_LSRA_STATS private: - Compiler* compiler; + Compiler* compiler; CompAllocator getAllocator(Compiler* comp) { return comp->getAllocator(CMK_LSRA); From 0b3da210a8b754521de29d40e01072fb1b01adc7 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 8 May 2023 11:23:29 -0700 Subject: [PATCH 11/15] Use popcount intrinsics --- src/coreclr/jit/compiler.hpp | 2 +- src/coreclr/jit/utils.cpp | 39 ++++++++++++------------------------ 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index e35d1d0cd666d..cec779f657ea8 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -116,7 +116,7 @@ inline bool genMaxOneBit(T value) template inline bool genExactlyOneBit(T value) { - return ((value != 0) && genMaxOneBit(value)); + return BitOperations::PopCount(value) == 1; } /***************************************************************************** diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index 3e5ad119411e2..f49547338afeb 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -2788,19 +2788,11 @@ uint32_t BitOperations::Log2(uint64_t value) uint32_t BitOperations::PopCount(uint32_t value) { #if defined(_MSC_VER) - // Inspired by the Stanford Bit Twiddling Hacks by Sean Eron Anderson: - // http://graphics.stanford.edu/~seander/bithacks.html - - const uint32_t c1 = 0x55555555u; - const uint32_t c2 = 0x33333333u; - const uint32_t c3 = 0x0F0F0F0Fu; - const uint32_t c4 = 0x01010101u; - - value -= (value >> 1) & c1; - value = (value & c2) + ((value >> 2) & c2); - value = (((value + (value >> 4)) & c3) * c4) >> 24; - - return value; +#ifdef HOST_ARM64 + return _CountOneBits(value); +#else + return __popcnt(value); +#endif #else int32_t result = __builtin_popcount(value); return static_cast(result); @@ -2819,19 +2811,14 @@ uint32_t BitOperations::PopCount(uint32_t value) uint32_t BitOperations::PopCount(uint64_t value) { #if defined(_MSC_VER) - // Inspired by the Stanford Bit Twiddling Hacks by Sean Eron Anderson: - // http://graphics.stanford.edu/~seander/bithacks.html - - const uint64_t c1 = 0x5555555555555555ull; - const uint64_t c2 = 0x3333333333333333ull; - const uint64_t c3 = 0x0F0F0F0F0F0F0F0Full; - const uint64_t c4 = 0x0101010101010101ull; - - value -= (value >> 1) & c1; - value = (value & c2) + ((value >> 2) & c2); - value = (((value + (value >> 4)) & c3) * c4) >> 56; - - return static_cast(value); +#ifdef HOST_ARM64 + return _CountOneBits64(value); +#elif defined(HOST_64BIT) + int64_t result = __popcnt64(value); + return static_cast(result); +#else + return __popcnt(value >> 32); +#endif #else int32_t result = __builtin_popcountll(value); return static_cast(result); From 813cb8483e399bd070af238269408ce89c49f638 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 8 May 2023 22:07:35 -0700 Subject: [PATCH 12/15] Revert "Use popcount intrinsics" This reverts commit 0b3da210a8b754521de29d40e01072fb1b01adc7. --- src/coreclr/jit/compiler.hpp | 2 +- src/coreclr/jit/utils.cpp | 39 ++++++++++++++++++++++++------------ 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index cec779f657ea8..e35d1d0cd666d 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -116,7 +116,7 @@ inline bool genMaxOneBit(T value) template inline bool genExactlyOneBit(T value) { - return BitOperations::PopCount(value) == 1; + return ((value != 0) && genMaxOneBit(value)); } /***************************************************************************** diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index f49547338afeb..3e5ad119411e2 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -2788,11 +2788,19 @@ uint32_t BitOperations::Log2(uint64_t value) uint32_t BitOperations::PopCount(uint32_t value) { #if defined(_MSC_VER) -#ifdef HOST_ARM64 - return _CountOneBits(value); -#else - return __popcnt(value); -#endif + // Inspired by the Stanford Bit Twiddling Hacks by Sean Eron Anderson: + // http://graphics.stanford.edu/~seander/bithacks.html + + const uint32_t c1 = 0x55555555u; + const uint32_t c2 = 0x33333333u; + const uint32_t c3 = 0x0F0F0F0Fu; + const uint32_t c4 = 0x01010101u; + + value -= (value >> 1) & c1; + value = (value & c2) + ((value >> 2) & c2); + value = (((value + (value >> 4)) & c3) * c4) >> 24; + + return value; #else int32_t result = __builtin_popcount(value); return static_cast(result); @@ -2811,14 +2819,19 @@ uint32_t BitOperations::PopCount(uint32_t value) uint32_t BitOperations::PopCount(uint64_t value) { #if defined(_MSC_VER) -#ifdef HOST_ARM64 - return _CountOneBits64(value); -#elif defined(HOST_64BIT) - int64_t result = __popcnt64(value); - return static_cast(result); -#else - return __popcnt(value >> 32); -#endif + // Inspired by the Stanford Bit Twiddling Hacks by Sean Eron Anderson: + // http://graphics.stanford.edu/~seander/bithacks.html + + const uint64_t c1 = 0x5555555555555555ull; + const uint64_t c2 = 0x3333333333333333ull; + const uint64_t c3 = 0x0F0F0F0F0F0F0F0Full; + const uint64_t c4 = 0x0101010101010101ull; + + value -= (value >> 1) & c1; + value = (value & c2) + ((value >> 2) & c2); + value = (((value + (value >> 4)) & c3) * c4) >> 56; + + return static_cast(value); #else int32_t result = __builtin_popcountll(value); return static_cast(result); From 7128104726b7c7c4c250b2c10e1c51fc2685faa8 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Mon, 8 May 2023 22:09:05 -0700 Subject: [PATCH 13/15] revert unintentional change from superpmi.py --- src/coreclr/scripts/superpmi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/scripts/superpmi.py b/src/coreclr/scripts/superpmi.py index 5dde1857d8823..4fb1c207b064b 100644 --- a/src/coreclr/scripts/superpmi.py +++ b/src/coreclr/scripts/superpmi.py @@ -1347,7 +1347,7 @@ def replay(self): common_flags = [ "-v", "ewi", # display errors, warnings, missing, jit info - #"-r", os.path.join(temp_location, "repro") # Repro name, create .mc repro files + "-r", os.path.join(temp_location, "repro") # Repro name, create .mc repro files ] if self.coreclr_args.altjit: From a79cf1ed9263838460a6046a84ad6e9169f6caa1 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 9 May 2023 14:30:35 -0700 Subject: [PATCH 14/15] Revert "Do not pass RegisterType for non-arm sarchitectures" This reverts commit 46d4d3d88704beca5f0817db3a6e48fc1a14dac6. --- src/coreclr/jit/lsra.cpp | 120 +++++++++++++++++----------------- src/coreclr/jit/lsra.h | 48 +++++++------- src/coreclr/jit/lsrabuild.cpp | 2 +- 3 files changed, 84 insertions(+), 86 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index f72a22bb1243e..3b49eba297426 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -309,7 +309,7 @@ regMaskTP LinearScan::getMatchingConstants(regMaskTP mask, Interval* currentInte return result; } -void LinearScan::clearNextIntervalRef(regNumber reg ARM_ARG(var_types regType)) +void LinearScan::clearNextIntervalRef(regNumber reg, var_types regType) { nextIntervalRef[reg] = MaxLocation; #ifdef TARGET_ARM @@ -322,7 +322,7 @@ void LinearScan::clearNextIntervalRef(regNumber reg ARM_ARG(var_types regType)) #endif } -void LinearScan::clearSpillCost(regNumber reg ARM_ARG(var_types regType)) +void LinearScan::clearSpillCost(regNumber reg, var_types regType) { spillCost[reg] = 0; #ifdef TARGET_ARM @@ -431,7 +431,7 @@ regMaskTP LinearScan::internalFloatRegCandidates() bool LinearScan::isFree(RegRecord* regRecord) { return ((regRecord->assignedInterval == nullptr || !regRecord->assignedInterval->isActive) && - !isRegBusy(regRecord->regNum ARM_ARG(regRecord->registerType))); + !isRegBusy(regRecord->regNum, regRecord->registerType)); } RegRecord* LinearScan::getRegisterRecord(regNumber regNum) @@ -586,7 +586,7 @@ bool LinearScan::conflictingFixedRegReference(regNumber regNum, RefPosition* ref LsraLocation refLocation = refPosition->nodeLocation; RegRecord* regRecord = getRegisterRecord(regNum); - if (isRegInUse(regNum ARM_ARG(refPosition->getInterval()->registerType)) && + if (isRegInUse(regNum, refPosition->getInterval()->registerType) && (regRecord->assignedInterval != refPosition->getInterval())) { return true; @@ -2788,7 +2788,7 @@ bool LinearScan::isMatchingConstant(RegRecord* physRegRecord, RefPosition* refPo return false; } Interval* interval = refPosition->getInterval(); - if (!interval->isConstant || !isRegConstant(physRegRecord->regNum ARM_ARG(interval->registerType))) + if (!interval->isConstant || !isRegConstant(physRegRecord->regNum, interval->registerType)) { return false; } @@ -3131,12 +3131,12 @@ bool LinearScan::isSpillCandidate(Interval* current, RefPosition* refPosition, R LsraLocation refLocation = refPosition->nodeLocation; // We shouldn't be calling this if we haven't already determined that the register is not // busy until the next kill. - assert(!isRegBusy(physRegRecord->regNum ARM_ARG(current->registerType))); + assert(!isRegBusy(physRegRecord->regNum, current->registerType)); // We should already have determined that the register isn't actively in use. #ifdef TARGET_ARM64 - assert(!isRegInUse(physRegRecord->regNum ARM_ARG(current->registerType)) || refPosition->needsConsecutive); + assert(!isRegInUse(physRegRecord->regNum, current->registerType) || refPosition->needsConsecutive); #else - assert(!isRegInUse(physRegRecord->regNum ARM_ARG(current->registerType))); + assert(!isRegInUse(physRegRecord->regNum, current->registerType)); #endif // We shouldn't be calling this if 'refPosition' is a fixed reference to this register. assert(!refPosition->isFixedRefOfRegMask(candidateBit)); @@ -3643,11 +3643,11 @@ void LinearScan::unassignPhysReg(RegRecord* regRec, RefPosition* spillRefPositio else #endif // TARGET_ARM { - clearNextIntervalRef(thisRegNum ARM_ARG(assignedInterval->registerType)); - clearSpillCost(thisRegNum ARM_ARG(assignedInterval->registerType)); + clearNextIntervalRef(thisRegNum, assignedInterval->registerType); + clearSpillCost(thisRegNum, assignedInterval->registerType); checkAndClearInterval(regRec, spillRefPosition); } - makeRegAvailable(regToUnassign ARM_ARG(assignedInterval->registerType)); + makeRegAvailable(regToUnassign, assignedInterval->registerType); RefPosition* nextRefPosition = nullptr; if (spillRefPosition != nullptr) @@ -3745,7 +3745,7 @@ void LinearScan::unassignPhysReg(RegRecord* regRec, RefPosition* spillRefPositio regRec->previousInterval = nullptr; if (regRec->assignedInterval->physReg != thisRegNum) { - clearNextIntervalRef(thisRegNum ARM_ARG(regRec->assignedInterval->registerType)); + clearNextIntervalRef(thisRegNum, regRec->assignedInterval->registerType); } else { @@ -3834,7 +3834,7 @@ void LinearScan::spillGCRefs(RefPosition* killRefPosition) { INDEBUG(killedRegs = true); unassignPhysReg(regRecord, assignedInterval->recentRefPosition); - makeRegAvailable(nextReg ARM_ARG(assignedInterval->registerType)); + makeRegAvailable(nextReg, assignedInterval->registerType); } } INDEBUG(dumpLsraAllocationEvent(killedRegs ? LSRA_EVENT_DONE_KILL_GC_REFS : LSRA_EVENT_NO_GC_KILLS, nullptr, REG_NA, @@ -4146,8 +4146,8 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) { RegRecord* physRegRecord = getRegisterRecord(reg); Interval* assignedInterval = physRegRecord->assignedInterval; - clearNextIntervalRef(reg ARM_ARG(physRegRecord->registerType)); - clearSpillCost(reg ARM_ARG(physRegRecord->registerType)); + clearNextIntervalRef(reg, physRegRecord->registerType); + clearSpillCost(reg, physRegRecord->registerType); if (assignedInterval != nullptr) { assert(assignedInterval->isConstant); @@ -4290,7 +4290,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) assert(targetReg != REG_STK); assert(interval->assignedReg != nullptr && interval->assignedReg->regNum == targetReg && interval->assignedReg->assignedInterval == interval); - liveRegs |= getRegMask(targetReg ARM_ARG(interval->registerType)); + liveRegs |= getRegMask(targetReg, interval->registerType); continue; } } @@ -4320,7 +4320,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) // likely to match other assignments this way. targetReg = interval->physReg; interval->isActive = true; - liveRegs |= getRegMask(targetReg ARM_ARG(interval->registerType)); + liveRegs |= getRegMask(targetReg, interval->registerType); INDEBUG(inactiveRegs |= genRegMask(targetReg)); setVarReg(inVarToRegMap, varIndex, targetReg); } @@ -4332,7 +4332,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) if (targetReg != REG_STK) { RegRecord* targetRegRecord = getRegisterRecord(targetReg); - liveRegs |= getRegMask(targetReg ARM_ARG(interval->registerType)); + liveRegs |= getRegMask(targetReg, interval->registerType); if (!allocationPassComplete) { updateNextIntervalRef(targetReg, interval); @@ -4395,7 +4395,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) RegRecord* physRegRecord = getRegisterRecord(reg); if ((liveRegs & genRegMask(reg)) == 0) { - makeRegAvailable(reg ARM_ARG(physRegRecord->registerType)); + makeRegAvailable(reg, physRegRecord->registerType); Interval* assignedInterval = physRegRecord->assignedInterval; if (assignedInterval != nullptr) @@ -4542,7 +4542,7 @@ void LinearScan::makeRegisterInactive(RegRecord* physRegRecord) assignedInterval->isActive = false; if (assignedInterval->isConstant) { - clearNextIntervalRef(physRegRecord->regNum ARM_ARG(assignedInterval->registerType)); + clearNextIntervalRef(physRegRecord->regNum, assignedInterval->registerType); } } } @@ -4571,8 +4571,8 @@ void LinearScan::makeRegisterInactive(RegRecord* physRegRecord) void LinearScan::freeRegister(RegRecord* physRegRecord) { Interval* assignedInterval = physRegRecord->assignedInterval; - makeRegAvailable(physRegRecord->regNum ARM_ARG(physRegRecord->registerType)); - clearSpillCost(physRegRecord->regNum ARM_ARG(physRegRecord->registerType)); + makeRegAvailable(physRegRecord->regNum, physRegRecord->registerType); + clearSpillCost(physRegRecord->regNum, physRegRecord->registerType); makeRegisterInactive(physRegRecord); if (assignedInterval != nullptr) @@ -4692,14 +4692,14 @@ void LinearScan::allocateRegisters() { updateNextIntervalRef(reg, interval); updateSpillCost(reg, interval); - setRegInUse(reg ARM_ARG(interval->registerType)); - INDEBUG(registersToDump |= getRegMask(reg ARM_ARG(interval->registerType))); + setRegInUse(reg, interval->registerType); + INDEBUG(registersToDump |= getRegMask(reg, interval->registerType)); } } else { - clearNextIntervalRef(reg ARM_ARG(physRegRecord->registerType)); - clearSpillCost(reg ARM_ARG(physRegRecord->registerType)); + clearNextIntervalRef(reg, physRegRecord->registerType); + clearSpillCost(reg, physRegRecord->registerType); } } @@ -4752,7 +4752,7 @@ void LinearScan::allocateRegisters() tempRegsToMakeInactive &= ~nextRegBit; regNumber nextReg = genRegNumFromMask(nextRegBit); RegRecord* regRecord = getRegisterRecord(nextReg); - clearSpillCost(regRecord->regNum ARM_ARG(regRecord->registerType)); + clearSpillCost(regRecord->regNum, regRecord->registerType); makeRegisterInactive(regRecord); } if (currentRefPosition.nodeLocation > prevLocation) @@ -4874,8 +4874,7 @@ void LinearScan::allocateRegisters() { continue; } - assert(assignedInterval->isConstant == - isRegConstant(reg ARM_ARG(assignedInterval->registerType))); + assert(assignedInterval->isConstant == isRegConstant(reg, assignedInterval->registerType)); if (assignedInterval->isActive) { // If this is not the register most recently allocated, it must be from a copyReg, @@ -4903,14 +4902,14 @@ void LinearScan::allocateRegisters() if (isAssignedReg) { assert(nextIntervalRef[reg] == assignedInterval->getNextRefLocation()); - assert(!isRegAvailable(reg ARM_ARG(assignedInterval->registerType))); + assert(!isRegAvailable(reg, assignedInterval->registerType)); assert((recentRefPosition == nullptr) || (spillCost[reg] == getSpillWeight(physRegRecord))); } else { assert((nextIntervalRef[reg] == MaxLocation) || - isRegBusy(reg ARM_ARG(assignedInterval->registerType))); + isRegBusy(reg, assignedInterval->registerType)); } } else @@ -4922,7 +4921,7 @@ void LinearScan::allocateRegisters() else { assert(nextIntervalRef[reg] == MaxLocation); - assert(isRegAvailable(reg ARM_ARG(assignedInterval->registerType))); + assert(isRegAvailable(reg, assignedInterval->registerType)); assert(spillCost[reg] == 0); } } @@ -4931,8 +4930,8 @@ void LinearScan::allocateRegisters() else { // Available registers should not hold constants - assert(isRegAvailable(reg ARM_ARG(physRegRecord->registerType))); - assert(!isRegConstant(reg ARM_ARG(physRegRecord->registerType))); + assert(isRegAvailable(reg, physRegRecord->registerType)); + assert(!isRegConstant(reg, physRegRecord->registerType)); assert(nextIntervalRef[reg] == MaxLocation); assert(spillCost[reg] == 0); } @@ -5045,7 +5044,7 @@ void LinearScan::allocateRegisters() { if (assignedInterval != nullptr && !assignedInterval->isActive && assignedInterval->isConstant) { - clearConstantReg(regRecord->regNum ARM_ARG(assignedInterval->registerType)); + clearConstantReg(regRecord->regNum, assignedInterval->registerType); regRecord->assignedInterval = nullptr; spillCost[regRecord->regNum] = 0; @@ -5069,8 +5068,8 @@ void LinearScan::allocateRegisters() if (assignedInterval != nullptr) { unassignPhysReg(regRecord, assignedInterval->recentRefPosition); - clearConstantReg(regRecord->regNum ARM_ARG(assignedInterval->registerType)); - makeRegAvailable(regRecord->regNum ARM_ARG(assignedInterval->registerType)); + clearConstantReg(regRecord->regNum, assignedInterval->registerType); + makeRegAvailable(regRecord->regNum, assignedInterval->registerType); } clearRegBusyUntilKill(regRecord->regNum); INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_KEPT_ALLOCATION, nullptr, regRecord->regNum)); @@ -5151,9 +5150,9 @@ void LinearScan::allocateRegisters() setIntervalAsSpilled(currentInterval); if (assignedRegister != REG_NA) { - clearNextIntervalRef(assignedRegister ARM_ARG(currentInterval->registerType)); - clearSpillCost(assignedRegister ARM_ARG(currentInterval->registerType)); - makeRegAvailable(assignedRegister ARM_ARG(currentInterval->registerType)); + clearNextIntervalRef(assignedRegister, currentInterval->registerType); + clearSpillCost(assignedRegister, currentInterval->registerType); + makeRegAvailable(assignedRegister, currentInterval->registerType); } } } @@ -5210,7 +5209,7 @@ void LinearScan::allocateRegisters() unassignPhysReg(regRecord, currentInterval->firstRefPosition); if (assignedInterval->isConstant) { - clearConstantReg(assignedRegister ARM_ARG(assignedInterval->registerType)); + clearConstantReg(assignedRegister, assignedInterval->registerType); } INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_NO_REG_ALLOCATED, currentInterval)); } @@ -5268,7 +5267,7 @@ void LinearScan::allocateRegisters() // special putarg_reg doesn't get spilled and re-allocated prior to // its use at the call node. This is ensured by marking physical reg // record as busy until next kill. - setRegBusyUntilKill(srcInterval->physReg ARM_ARG(srcInterval->registerType)); + setRegBusyUntilKill(srcInterval->physReg, srcInterval->registerType); } else { @@ -5310,7 +5309,7 @@ void LinearScan::allocateRegisters() else { currentInterval->isActive = true; - setRegInUse(assignedRegister ARM_ARG(currentInterval->registerType)); + setRegInUse(assignedRegister, currentInterval->registerType); updateSpillCost(assignedRegister, currentInterval); } updateNextIntervalRef(assignedRegister, currentInterval); @@ -5404,7 +5403,7 @@ void LinearScan::allocateRegisters() RegRecord* physRegRecord = getRegisterRecord(assignedRegister); assert((assignedRegBit == currentRefPosition.registerAssignment) || (physRegRecord->assignedInterval == currentInterval) || - !isRegInUse(assignedRegister ARM_ARG(currentInterval->registerType))); + !isRegInUse(assignedRegister, currentInterval->registerType)); if (conflictingFixedRegReference(assignedRegister, ¤tRefPosition)) { // We may have already reassigned the register to the conflicting reference. @@ -5412,7 +5411,7 @@ void LinearScan::allocateRegisters() if (physRegRecord->assignedInterval == currentInterval) { unassignPhysRegNoSpill(physRegRecord); - clearConstantReg(assignedRegister ARM_ARG(currentInterval->registerType)); + clearConstantReg(assignedRegister, currentInterval->registerType); } currentRefPosition.moveReg = true; assignedRegister = REG_NA; @@ -5445,10 +5444,9 @@ void LinearScan::allocateRegisters() if (copyReg != assignedRegister) { - lastAllocatedRefPosition = ¤tRefPosition; - regMaskTP copyRegMask = getRegMask(copyReg ARM_ARG(currentInterval->registerType)); - regMaskTP assignedRegMask = - getRegMask(assignedRegister ARM_ARG(currentInterval->registerType)); + lastAllocatedRefPosition = ¤tRefPosition; + regMaskTP copyRegMask = getRegMask(copyReg, currentInterval->registerType); + regMaskTP assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); if ((consecutiveRegsInUseThisLocation & assignedRegMask) != RBM_NONE) { @@ -5483,8 +5481,8 @@ void LinearScan::allocateRegisters() currentRefPosition.moveReg = true; currentRefPosition.copyReg = false; } - clearNextIntervalRef(copyReg ARM_ARG(currentInterval->registerType)); - clearSpillCost(copyReg ARM_ARG(currentInterval->registerType)); + clearNextIntervalRef(copyReg, currentInterval->registerType); + clearSpillCost(copyReg, currentInterval->registerType); updateNextIntervalRef(assignedRegister, currentInterval); updateSpillCost(assignedRegister, currentInterval); } @@ -5544,8 +5542,8 @@ void LinearScan::allocateRegisters() } lastAllocatedRefPosition = ¤tRefPosition; - regMaskTP copyRegMask = getRegMask(copyReg ARM_ARG(currentInterval->registerType)); - regMaskTP assignedRegMask = getRegMask(assignedRegister ARM_ARG(currentInterval->registerType)); + regMaskTP copyRegMask = getRegMask(copyReg, currentInterval->registerType); + regMaskTP assignedRegMask = getRegMask(assignedRegister, currentInterval->registerType); #ifdef TARGET_ARM64 if (hasConsecutiveRegister && currentRefPosition.needsConsecutive) @@ -5590,8 +5588,8 @@ void LinearScan::allocateRegisters() currentRefPosition.moveReg = true; currentRefPosition.copyReg = false; } - clearNextIntervalRef(copyReg ARM_ARG(currentInterval->registerType)); - clearSpillCost(copyReg ARM_ARG(currentInterval->registerType)); + clearNextIntervalRef(copyReg, currentInterval->registerType); + clearSpillCost(copyReg, currentInterval->registerType); updateNextIntervalRef(assignedRegister, currentInterval); updateSpillCost(assignedRegister, currentInterval); continue; @@ -5599,7 +5597,7 @@ void LinearScan::allocateRegisters() else { INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_NEEDS_NEW_REG, nullptr, assignedRegister)); - regsToFree |= getRegMask(assignedRegister ARM_ARG(currentInterval->registerType)); + regsToFree |= getRegMask(assignedRegister, currentInterval->registerType); // We want a new register, but we don't want this to be considered a spill. assignedRegister = REG_NA; if (physRegRecord->assignedInterval == currentInterval) @@ -5767,7 +5765,7 @@ void LinearScan::allocateRegisters() if (assignedRegister != REG_NA) { assignedRegBit = genRegMask(assignedRegister); - regMaskTP regMask = getRegMask(assignedRegister ARM_ARG(currentInterval->registerType)); + regMaskTP regMask = getRegMask(assignedRegister, currentInterval->registerType); regsInUseThisLocation |= regMask; if (currentRefPosition.delayRegFree) { @@ -6060,14 +6058,14 @@ void LinearScan::updateAssignedInterval(RegRecord* reg, Interval* interval ARM_A } #endif reg->assignedInterval = interval; - setRegInUse(reg->regNum ARM_ARG(interval->registerType)); + setRegInUse(reg->regNum, interval->registerType); if (interval->isConstant) { - setConstantReg(reg->regNum ARM_ARG(interval->registerType)); + setConstantReg(reg->regNum, interval->registerType); } else { - clearConstantReg(reg->regNum ARM_ARG(interval->registerType)); + clearConstantReg(reg->regNum, interval->registerType); } updateNextIntervalRef(reg->regNum, interval); updateSpillCost(reg->regNum, interval); @@ -10118,7 +10116,7 @@ void LinearScan::dumpLsraAllocationEvent( } if ((interval != nullptr) && (reg != REG_NA) && (reg != REG_STK)) { - registersToDump |= getRegMask(reg ARM_ARG(interval->registerType)); + registersToDump |= getRegMask(reg, interval->registerType); dumpRegRecordTitleIfNeeded(); } diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 43a6009d9db35..edfd3f3af5cd0 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1723,7 +1723,7 @@ class LinearScan : public LinearScanInterface //----------------------------------------------------------------------- regMaskTP m_AvailableRegs; - regNumber getRegForType(regNumber reg ARM_ARG(var_types regType)) + regNumber getRegForType(regNumber reg, var_types regType) { #ifdef TARGET_ARM if ((regType == TYP_DOUBLE) && !genIsValidDoubleReg(reg)) @@ -1734,9 +1734,9 @@ class LinearScan : public LinearScanInterface return reg; } - regMaskTP getRegMask(regNumber reg ARM_ARG(var_types regType)) + regMaskTP getRegMask(regNumber reg, var_types regType) { - reg = getRegForType(reg ARM_ARG(regType)); + reg = getRegForType(reg, regType); regMaskTP regMask = genRegMask(reg); #ifdef TARGET_ARM if (regType == TYP_DOUBLE) @@ -1754,34 +1754,34 @@ class LinearScan : public LinearScanInterface m_RegistersWithConstants = RBM_NONE; } - bool isRegAvailable(regNumber reg ARM_ARG(var_types regType)) + bool isRegAvailable(regNumber reg, var_types regType) { - regMaskTP regMask = getRegMask(reg ARM_ARG(regType)); + regMaskTP regMask = getRegMask(reg, regType); return (m_AvailableRegs & regMask) == regMask; } void setRegsInUse(regMaskTP regMask) { m_AvailableRegs &= ~regMask; } - void setRegInUse(regNumber reg ARM_ARG(var_types regType)) + void setRegInUse(regNumber reg, var_types regType) { - regMaskTP regMask = getRegMask(reg ARM_ARG(regType)); + regMaskTP regMask = getRegMask(reg, regType); setRegsInUse(regMask); } void makeRegsAvailable(regMaskTP regMask) { m_AvailableRegs |= regMask; } - void makeRegAvailable(regNumber reg ARM_ARG(var_types regType)) + void makeRegAvailable(regNumber reg, var_types regType) { - regMaskTP regMask = getRegMask(reg ARM_ARG(regType)); + regMaskTP regMask = getRegMask(reg, regType); makeRegsAvailable(regMask); } - void clearNextIntervalRef(regNumber reg ARM_ARG(var_types regType)); + void clearNextIntervalRef(regNumber reg, var_types regType); void updateNextIntervalRef(regNumber reg, Interval* interval); - void clearSpillCost(regNumber reg ARM_ARG(var_types regType)); + void clearSpillCost(regNumber reg, var_types regType); void updateSpillCost(regNumber reg, Interval* interval); FORCEINLINE void updateRegsFreeBusyState(RefPosition& refPosition, @@ -1791,18 +1791,18 @@ class LinearScan : public LinearScanInterface DEBUG_ARG(regNumber assignedReg)); regMaskTP m_RegistersWithConstants; - void clearConstantReg(regNumber reg ARM_ARG(var_types regType)) + void clearConstantReg(regNumber reg, var_types regType) { - m_RegistersWithConstants &= ~getRegMask(reg ARM_ARG(regType)); + m_RegistersWithConstants &= ~getRegMask(reg, regType); } - void setConstantReg(regNumber reg ARM_ARG(var_types regType)) + void setConstantReg(regNumber reg, var_types regType) { - m_RegistersWithConstants |= getRegMask(reg ARM_ARG(regType)); + m_RegistersWithConstants |= getRegMask(reg, regType); } - bool isRegConstant(regNumber reg ARM_ARG(var_types regType)) + bool isRegConstant(regNumber reg, var_types regType) { - reg = getRegForType(reg ARM_ARG(regType)); - regMaskTP regMask = getRegMask(reg ARM_ARG(regType)); + reg = getRegForType(reg, regType); + regMaskTP regMask = getRegMask(reg, regType); return (m_RegistersWithConstants & regMask) == regMask; } regMaskTP getMatchingConstants(regMaskTP mask, Interval* currentInterval, RefPosition* refPosition); @@ -1842,23 +1842,23 @@ class LinearScan : public LinearScanInterface #ifdef TARGET_ARM64 regMaskTP consecutiveRegsInUseThisLocation; #endif - bool isRegBusy(regNumber reg ARM_ARG(var_types regType)) + bool isRegBusy(regNumber reg, var_types regType) { - regMaskTP regMask = getRegMask(reg ARM_ARG(regType)); + regMaskTP regMask = getRegMask(reg, regType); return (regsBusyUntilKill & regMask) != RBM_NONE; } - void setRegBusyUntilKill(regNumber reg ARM_ARG(var_types regType)) + void setRegBusyUntilKill(regNumber reg, var_types regType) { - regsBusyUntilKill |= getRegMask(reg ARM_ARG(regType)); + regsBusyUntilKill |= getRegMask(reg, regType); } void clearRegBusyUntilKill(regNumber reg) { regsBusyUntilKill &= ~genRegMask(reg); } - bool isRegInUse(regNumber reg ARM_ARG(var_types regType)) + bool isRegInUse(regNumber reg, var_types regType) { - regMaskTP regMask = getRegMask(reg ARM_ARG(regType)); + regMaskTP regMask = getRegMask(reg, regType); return (regsInUseThisLocation & regMask) != RBM_NONE; } diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 4ebee0b810f33..0f11acdadb701 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -2273,7 +2273,7 @@ void LinearScan::buildIntervals() assert(inArgReg < REG_COUNT); mask = genRegMask(inArgReg); assignPhysReg(inArgReg, interval); - INDEBUG(registersToDump |= getRegMask(inArgReg ARM_ARG(interval->registerType))); + INDEBUG(registersToDump |= getRegMask(inArgReg, interval->registerType)); } RefPosition* pos = newRefPosition(interval, MinLocation, RefTypeParamDef, nullptr, mask); pos->setRegOptional(true); From 7dd4d462b10012a2aed1b0bdc9263fabf3184aa6 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Tue, 9 May 2023 14:41:25 -0700 Subject: [PATCH 15/15] fix the merge conflicts --- src/coreclr/jit/lsra.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 3b49eba297426..262a5b87aef8b 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -6011,8 +6011,8 @@ void LinearScan::clearAssignedInterval(RegRecord* reg ARM_ARG(RegisterType regTy #endif // TARGET_ARM reg->assignedInterval = nullptr; - clearNextIntervalRef(reg->regNum ARM_ARG(reg->registerType)); - clearSpillCost(reg->regNum ARM_ARG(reg->registerType)); + clearNextIntervalRef(reg->regNum, reg->registerType); + clearSpillCost(reg->regNum, reg->registerType); } //-----------------------------------------------------------------------------