From 23e15a51b8c45df8c5efbd5b80d577eaef981edb Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 29 May 2024 05:55:56 -0700 Subject: [PATCH] LSRA: Make genRegMask() return `regMaskTP` (#102783) * genRegMask * Make genRegMask() return regMaskTP, introduce genSingleTypeRegMask() for LSRA * Make allFloat,allMask regMaskTP instead of `SingleTypeRegSet` so no affect on non-LSRA code * jit format * fix build errors * fix loongarch and risc * fix a typo * move more `genSingleTypeRegMask()` * jit format * Make genRegMask() use genSingleType*() * fix build errors * jit format --- src/coreclr/jit/codegeninterface.h | 16 ++-- src/coreclr/jit/compiler.cpp | 4 +- src/coreclr/jit/compiler.h | 30 +++--- src/coreclr/jit/lsra.cpp | 111 +++++++++++++--------- src/coreclr/jit/lsra.h | 33 +++---- src/coreclr/jit/lsraarm.cpp | 4 +- src/coreclr/jit/lsraarm64.cpp | 2 +- src/coreclr/jit/lsraarmarch.cpp | 23 ++--- src/coreclr/jit/lsrabuild.cpp | 26 ++++-- src/coreclr/jit/lsraloongarch64.cpp | 10 +- src/coreclr/jit/lsrariscv64.cpp | 13 +-- src/coreclr/jit/lsraxarch.cpp | 4 +- src/coreclr/jit/regMaskTPOps.cpp | 2 +- src/coreclr/jit/target.h | 138 ++++++++++++++++++++-------- 14 files changed, 254 insertions(+), 162 deletions(-) diff --git a/src/coreclr/jit/codegeninterface.h b/src/coreclr/jit/codegeninterface.h index a025285cbc091..608c72c22d48d 100644 --- a/src/coreclr/jit/codegeninterface.h +++ b/src/coreclr/jit/codegeninterface.h @@ -75,31 +75,31 @@ class CodeGenInterface } #if defined(TARGET_AMD64) - SingleTypeRegSet rbmAllFloat; - SingleTypeRegSet rbmFltCalleeTrash; + regMaskTP rbmAllFloat; + regMaskTP rbmFltCalleeTrash; - FORCEINLINE SingleTypeRegSet get_RBM_ALLFLOAT() const + FORCEINLINE regMaskTP get_RBM_ALLFLOAT() const { return this->rbmAllFloat; } - FORCEINLINE SingleTypeRegSet get_RBM_FLT_CALLEE_TRASH() const + FORCEINLINE regMaskTP get_RBM_FLT_CALLEE_TRASH() const { return this->rbmFltCalleeTrash; } #endif // TARGET_AMD64 #if defined(TARGET_XARCH) - SingleTypeRegSet rbmAllMask; - SingleTypeRegSet rbmMskCalleeTrash; + regMaskTP rbmAllMask; + regMaskTP rbmMskCalleeTrash; // Call this function after the equivalent fields in Compiler have been initialized. void CopyRegisterInfo(); - FORCEINLINE SingleTypeRegSet get_RBM_ALLMASK() const + FORCEINLINE regMaskTP get_RBM_ALLMASK() const { return this->rbmAllMask; } - FORCEINLINE SingleTypeRegSet get_RBM_MSK_CALLEE_TRASH() const + FORCEINLINE regMaskTP get_RBM_MSK_CALLEE_TRASH() const { return this->rbmMskCalleeTrash; } diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 1b5592a5b59e5..54be15ed7f20b 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -3485,12 +3485,12 @@ void Compiler::compInitOptions(JitFlags* jitFlags) // Make sure we copy the register info and initialize the // trash regs after the underlying fields are initialized - const SingleTypeRegSet vtCalleeTrashRegs[TYP_COUNT]{ + const regMaskTP vtCalleeTrashRegs[TYP_COUNT]{ #define DEF_TP(tn, nm, jitType, sz, sze, asze, st, al, regTyp, regFld, csr, ctr, tf) ctr, #include "typelist.h" #undef DEF_TP }; - memcpy(varTypeCalleeTrashRegs, vtCalleeTrashRegs, sizeof(SingleTypeRegSet) * TYP_COUNT); + memcpy(varTypeCalleeTrashRegs, vtCalleeTrashRegs, sizeof(regMaskTP) * TYP_COUNT); codeGen->CopyRegisterInfo(); #endif // TARGET_XARCH diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 76df03352c55a..96c8c2b500bbf 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -11246,8 +11246,8 @@ class Compiler // // Users of these values need to define four accessor functions: // - // SingleTypeRegSet get_RBM_ALLFLOAT(); - // SingleTypeRegSet get_RBM_FLT_CALLEE_TRASH(); + // regMaskTP get_RBM_ALLFLOAT(); + // regMaskTP get_RBM_FLT_CALLEE_TRASH(); // unsigned get_CNT_CALLEE_TRASH_FLOAT(); // unsigned get_AVAILABLE_REG_COUNT(); // @@ -11256,16 +11256,16 @@ class Compiler // This was done to avoid polluting all `targetXXX.h` macro definitions with a compiler parameter, where only // TARGET_AMD64 requires one. // - SingleTypeRegSet rbmAllFloat; - SingleTypeRegSet rbmFltCalleeTrash; - unsigned cntCalleeTrashFloat; + regMaskTP rbmAllFloat; + regMaskTP rbmFltCalleeTrash; + unsigned cntCalleeTrashFloat; public: - FORCEINLINE SingleTypeRegSet get_RBM_ALLFLOAT() const + FORCEINLINE regMaskTP get_RBM_ALLFLOAT() const { return this->rbmAllFloat; } - FORCEINLINE SingleTypeRegSet get_RBM_FLT_CALLEE_TRASH() const + FORCEINLINE regMaskTP get_RBM_FLT_CALLEE_TRASH() const { return this->rbmFltCalleeTrash; } @@ -11284,8 +11284,8 @@ class Compiler // // Users of these values need to define four accessor functions: // - // SingleTypeRegSet get_RBM_ALLMASK(); - // SingleTypeRegSet get_RBM_MSK_CALLEE_TRASH(); + // regMaskTP get_RBM_ALLMASK(); + // regMaskTP get_RBM_MSK_CALLEE_TRASH(); // unsigned get_CNT_CALLEE_TRASH_MASK(); // unsigned get_AVAILABLE_REG_COUNT(); // @@ -11294,17 +11294,17 @@ class Compiler // This was done to avoid polluting all `targetXXX.h` macro definitions with a compiler parameter, where only // TARGET_XARCH requires one. // - SingleTypeRegSet rbmAllMask; - SingleTypeRegSet rbmMskCalleeTrash; - unsigned cntCalleeTrashMask; - SingleTypeRegSet varTypeCalleeTrashRegs[TYP_COUNT]; + regMaskTP rbmAllMask; + regMaskTP rbmMskCalleeTrash; + unsigned cntCalleeTrashMask; + regMaskTP varTypeCalleeTrashRegs[TYP_COUNT]; public: - FORCEINLINE SingleTypeRegSet get_RBM_ALLMASK() const + FORCEINLINE regMaskTP get_RBM_ALLMASK() const { return this->rbmAllMask; } - FORCEINLINE SingleTypeRegSet get_RBM_MSK_CALLEE_TRASH() const + FORCEINLINE regMaskTP get_RBM_MSK_CALLEE_TRASH() const { return this->rbmMskCalleeTrash; } diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index b642d59bca05a..08c30882d59d0 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -278,7 +278,7 @@ void LinearScan::updateNextFixedRef(RegRecord* regRecord, RefPosition* nextRefPo RefPosition* kill = nextKill; while ((kill != nullptr) && (kill->nodeLocation < nextLocation)) { - if ((kill->registerAssignment & genRegMask(regRecord->regNum)) != RBM_NONE) + if ((kill->registerAssignment & genSingleTypeRegMask(regRecord->regNum)) != RBM_NONE) { nextLocation = kill->nodeLocation; break; @@ -309,7 +309,7 @@ SingleTypeRegSet LinearScan::getMatchingConstants(SingleTypeRegSet mask, while (candidates != RBM_NONE) { regNumber regNum = genFirstRegNumFromMask(candidates); - SingleTypeRegSet candidateBit = genRegMask(regNum); + SingleTypeRegSet candidateBit = genSingleTypeRegMask(regNum); candidates ^= candidateBit; RegRecord* physRegRecord = getRegisterRecord(regNum); @@ -448,7 +448,11 @@ SingleTypeRegSet LinearScan::internalFloatRegCandidates() } else { +#ifdef TARGET_AMD64 + return RBM_FLT_CALLEE_TRASH.GetFloatRegSet(); +#else return RBM_FLT_CALLEE_TRASH; +#endif // TARGET_AMD64 } } @@ -597,7 +601,12 @@ SingleTypeRegSet LinearScan::stressLimitRegs(RefPosition* refPosition, RegisterT case LSRA_LIMIT_CALLER: { +#ifdef TARGET_XARCH + mask = getConstrainedRegMask(refPosition, regType, mask, RBM_CALLEE_TRASH.GetRegSetForType(regType), + minRegCount); +#else mask = getConstrainedRegMask(refPosition, regType, mask, RBM_CALLEE_TRASH, minRegCount); +#endif // TARGET_AMD64 } break; @@ -658,7 +667,7 @@ SingleTypeRegSet LinearScan::stressLimitRegs(RefPosition* refPosition, RegisterT bool LinearScan::conflictingFixedRegReference(regNumber regNum, RefPosition* refPosition) { // Is this a fixed reference of this register? If so, there is no conflict. - if (refPosition->isFixedRefOfRegMask(genRegMask(regNum))) + if (refPosition->isFixedRefOfRegMask(genSingleTypeRegMask(regNum))) { return false; } @@ -784,7 +793,7 @@ LinearScan::LinearScan(Compiler* theCompiler) #if defined(TARGET_XARCH) rbmAllMask = compiler->rbmAllMask; rbmMskCalleeTrash = compiler->rbmMskCalleeTrash; - memcpy(varTypeCalleeTrashRegs, compiler->varTypeCalleeTrashRegs, sizeof(SingleTypeRegSet) * TYP_COUNT); + memcpy(varTypeCalleeTrashRegs, compiler->varTypeCalleeTrashRegs, sizeof(regMaskTP) * TYP_COUNT); if (!compiler->canUseEvexEncoding()) { @@ -848,9 +857,17 @@ LinearScan::LinearScan(Compiler* theCompiler) availableIntRegs &= ~RBM_FPBASE; #endif // ETW_EBP_FRAMED +#ifdef TARGET_AMD64 + availableFloatRegs = RBM_ALLFLOAT.GetFloatRegSet(); + availableDoubleRegs = RBM_ALLDOUBLE.GetFloatRegSet(); +#else availableFloatRegs = RBM_ALLFLOAT; availableDoubleRegs = RBM_ALLDOUBLE; -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) +#endif + +#if defined(TARGET_XARCH) + availableMaskRegs = RBM_ALLMASK.GetPredicateRegSet(); +#elif defined(TARGET_ARM64) availableMaskRegs = RBM_ALLMASK; #endif @@ -3289,8 +3306,8 @@ bool LinearScan::isRefPositionActive(RefPosition* refPosition, LsraLocation refL // bool LinearScan::isSpillCandidate(Interval* current, RefPosition* refPosition, RegRecord* physRegRecord) { - regMaskTP candidateBit = genRegMask(physRegRecord->regNum); - LsraLocation refLocation = refPosition->nodeLocation; + SingleTypeRegSet candidateBit = genSingleTypeRegMask(physRegRecord->regNum); + LsraLocation refLocation = refPosition->nodeLocation; // We shouldn't be calling this if we haven't already determined that the register is not // busy until the next kill. assert(!isRegBusy(physRegRecord->regNum, current->registerType)); @@ -3507,7 +3524,7 @@ void LinearScan::checkAndAssignInterval(RegRecord* regRec, Interval* interval) // Assign the given physical register interval to the given interval void LinearScan::assignPhysReg(RegRecord* regRec, Interval* interval) { - SingleTypeRegSet assignedRegMask = genRegMask(regRec->regNum); + SingleTypeRegSet assignedRegMask = genSingleTypeRegMask(regRec->regNum); compiler->codeGen->regSet.rsSetRegsModified(assignedRegMask DEBUGARG(true)); interval->assignedReg = regRec; @@ -4645,7 +4662,7 @@ void LinearScan::processBlockStartLocations(BasicBlock* currentBlock) assignPhysReg(targetRegRecord, interval); } if (interval->recentRefPosition != nullptr && !interval->recentRefPosition->copyReg && - interval->recentRefPosition->registerAssignment != genRegMask(targetReg)) + interval->recentRefPosition->registerAssignment != genSingleTypeRegMask(targetReg)) { interval->getNextRefPosition()->outOfOrder = true; } @@ -5265,7 +5282,7 @@ void LinearScan::allocateRegistersMinimal() if (assignedRegister != REG_NA) { isInRegister = true; - assignedRegBit = genRegMask(assignedRegister); + assignedRegBit = genSingleTypeRegMask(assignedRegister); if (!currentInterval->isActive) { assert(!RefTypeIsUse(refType)); @@ -5309,7 +5326,7 @@ void LinearScan::allocateRegistersMinimal() setIntervalAsSplit(currentInterval); INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_MOVE_REG, currentInterval, assignedRegister)); } - else if ((genRegMask(assignedRegister) & currentRefPosition.registerAssignment) != 0) + else if ((genSingleTypeRegMask(assignedRegister) & currentRefPosition.registerAssignment) != 0) { currentRefPosition.registerAssignment = assignedRegBit; if (!currentInterval->isActive) @@ -5402,7 +5419,8 @@ void LinearScan::allocateRegistersMinimal() if (currentRefPosition.isFixedRegRef && !currentInterval->isActive && (currentInterval->assignedReg != nullptr) && (currentInterval->assignedReg->assignedInterval == currentInterval) && - (genRegMask(currentInterval->assignedReg->regNum) != currentRefPosition.registerAssignment)) + (genSingleTypeRegMask(currentInterval->assignedReg->regNum) != + currentRefPosition.registerAssignment)) { unassignPhysReg(currentInterval->assignedReg, nullptr); } @@ -5450,7 +5468,7 @@ void LinearScan::allocateRegistersMinimal() // If we allocated a register, record it if (assignedRegister != REG_NA) { - assignedRegBit = genRegMask(assignedRegister); + assignedRegBit = genSingleTypeRegMask(assignedRegister); regMaskTP regMask = getRegMask(assignedRegister, currentInterval->registerType); regsInUseThisLocation |= regMask; if (currentRefPosition.delayRegFree) @@ -6153,7 +6171,7 @@ void LinearScan::allocateRegisters() // kill would lead to spill of source but not the putarg_reg if it were treated // as special. if (srcInterval->isActive && - genRegMask(srcInterval->physReg) == currentRefPosition.registerAssignment && + genSingleTypeRegMask(srcInterval->physReg) == currentRefPosition.registerAssignment && currentInterval->getNextRefLocation() == nextFixedRef[srcInterval->physReg]) { assert(physRegRecord->regNum == srcInterval->physReg); @@ -6197,7 +6215,7 @@ void LinearScan::allocateRegisters() if (assignedRegister != REG_NA) { isInRegister = true; - assignedRegBit = genRegMask(assignedRegister); + assignedRegBit = genSingleTypeRegMask(assignedRegister); if (!currentInterval->isActive) { // If this is a use, it must have started the block on the stack, but the register @@ -6239,9 +6257,9 @@ void LinearScan::allocateRegisters() // it might be beneficial to keep it in this reg for PART of the lifetime if (currentInterval->isLocalVar) { - regMaskTP preferences = currentInterval->registerPreferences; - bool keepAssignment = true; - bool matchesPreferences = (preferences & genRegMask(assignedRegister)) != RBM_NONE; + SingleTypeRegSet preferences = currentInterval->registerPreferences; + bool keepAssignment = true; + bool matchesPreferences = (preferences & genSingleTypeRegMask(assignedRegister)) != RBM_NONE; // Will the assigned register cover the lifetime? If not, does it at least // meet the preferences for the next RefPosition? @@ -6322,7 +6340,7 @@ void LinearScan::allocateRegisters() setIntervalAsSplit(currentInterval); INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_MOVE_REG, currentInterval, assignedRegister)); } - else if ((genRegMask(assignedRegister) & currentRefPosition.registerAssignment) != 0) + else if ((genSingleTypeRegMask(assignedRegister) & currentRefPosition.registerAssignment) != 0) { #ifdef TARGET_ARM64 if (hasConsecutiveRegister && currentRefPosition.isFirstRefPositionOfConsecutiveRegisters()) @@ -6600,7 +6618,8 @@ void LinearScan::allocateRegisters() if (currentRefPosition.isFixedRegRef && !currentInterval->isActive && (currentInterval->assignedReg != nullptr) && (currentInterval->assignedReg->assignedInterval == currentInterval) && - (genRegMask(currentInterval->assignedReg->regNum) != currentRefPosition.registerAssignment)) + (genSingleTypeRegMask(currentInterval->assignedReg->regNum) != + currentRefPosition.registerAssignment)) { unassignPhysReg(currentInterval->assignedReg, nullptr); } @@ -6668,7 +6687,7 @@ void LinearScan::allocateRegisters() // If we allocated a register, record it if (assignedRegister != REG_NA) { - assignedRegBit = genRegMask(assignedRegister); + assignedRegBit = genSingleTypeRegMask(assignedRegister); regMaskTP regMask = getRegMask(assignedRegister, currentInterval->registerType); regsInUseThisLocation |= regMask; if (currentRefPosition.delayRegFree) @@ -8451,7 +8470,7 @@ void LinearScan::resolveRegisters() varDsc->lvOnFrame = false; } #ifdef DEBUG - regMaskTP registerAssignment = genRegMask(varDsc->GetRegNum()); + regMaskTP registerAssignment = genSingleTypeRegMask(varDsc->GetRegNum()); assert(!interval->isSpilled && !interval->isSplit); RefPosition* refPosition = interval->firstRefPosition; assert(refPosition != nullptr); @@ -8731,7 +8750,7 @@ regNumber LinearScan::getTempRegForResolution(BasicBlock* fromBlock, assert(fromReg != REG_NA); if (fromReg != REG_STK) { - freeRegs &= ~genRegMask(fromReg ARM_ARG(getIntervalForLocalVar(varIndex)->registerType)); + freeRegs &= ~genSingleTypeRegMask(fromReg ARM_ARG(getIntervalForLocalVar(varIndex)->registerType)); } if (toBlock != nullptr) @@ -8740,7 +8759,7 @@ regNumber LinearScan::getTempRegForResolution(BasicBlock* fromBlock, assert(toReg != REG_NA); if (toReg != REG_STK) { - freeRegs &= ~genRegMask(toReg ARM_ARG(getIntervalForLocalVar(varIndex)->registerType)); + freeRegs &= ~genSingleTypeRegMask(toReg ARM_ARG(getIntervalForLocalVar(varIndex)->registerType)); } } } @@ -8759,7 +8778,7 @@ regNumber LinearScan::getTempRegForResolution(BasicBlock* fromBlock, assert(reg != REG_NA); if (reg != REG_STK) { - freeRegs &= ~genRegMask(reg ARM_ARG(getIntervalForLocalVar(varIndex)->registerType)); + freeRegs &= ~genSingleTypeRegMask(reg ARM_ARG(getIntervalForLocalVar(varIndex)->registerType)); } } } @@ -8782,7 +8801,11 @@ regNumber LinearScan::getTempRegForResolution(BasicBlock* fromBlock, // Prefer a callee-trashed register if possible to prevent new prolog/epilog saves/restores. if ((freeRegs & RBM_CALLEE_TRASH) != 0) { +#ifdef TARGET_XARCH + freeRegs &= RBM_CALLEE_TRASH.GetRegSetForType(type); +#else freeRegs &= RBM_CALLEE_TRASH; +#endif } regNumber tempReg = genRegNumFromMask(genFindLowestBit(freeRegs)); @@ -9010,17 +9033,17 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) noway_assert(op1 != nullptr && op2 != nullptr); assert(op1->GetRegNum() != REG_NA && op2->GetRegNum() != REG_NA); // No floating point values, so no need to worry about the register type - // (i.e. for ARM32, where we used the genRegMask overload with a type). + // (i.e. for ARM32, where we used the genSingleTypeRegMask overload with a type). assert(varTypeIsIntegralOrI(op1) && varTypeIsIntegralOrI(op2)); - consumedRegs |= genRegMask(op1->GetRegNum()); - consumedRegs |= genRegMask(op2->GetRegNum()); + consumedRegs |= genSingleTypeRegMask(op1->GetRegNum()); + consumedRegs |= genSingleTypeRegMask(op2->GetRegNum()); // Special handling for GT_COPY to not resolve into the source // of switch's operand. if (op1->OperIs(GT_COPY)) { GenTree* srcOp1 = op1->gtGetOp1(); - consumedRegs |= genRegMask(srcOp1->GetRegNum()); + consumedRegs |= genSingleTypeRegMask(srcOp1->GetRegNum()); } else if (op1->IsLocal()) { @@ -9030,7 +9053,7 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) if (op2->OperIs(GT_COPY)) { GenTree* srcOp2 = op2->gtGetOp1(); - consumedRegs |= genRegMask(srcOp2->GetRegNum()); + consumedRegs |= genSingleTypeRegMask(srcOp2->GetRegNum()); } else if (op2->IsLocal()) { @@ -9058,12 +9081,12 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) if (lastNode->OperIs(GT_JTRUE, GT_JCMP, GT_JTEST)) { GenTree* op = lastNode->gtGetOp1(); - consumedRegs |= genRegMask(op->GetRegNum()); + consumedRegs |= genSingleTypeRegMask(op->GetRegNum()); if (op->OperIs(GT_COPY)) { GenTree* srcOp = op->gtGetOp1(); - consumedRegs |= genRegMask(srcOp->GetRegNum()); + consumedRegs |= genSingleTypeRegMask(srcOp->GetRegNum()); } else if (op->IsLocal()) { @@ -9074,12 +9097,12 @@ void LinearScan::handleOutgoingCriticalEdges(BasicBlock* block) if (lastNode->OperIs(GT_JCMP, GT_JTEST) && !lastNode->gtGetOp2()->isContained()) { op = lastNode->gtGetOp2(); - consumedRegs |= genRegMask(op->GetRegNum()); + consumedRegs |= genSingleTypeRegMask(op->GetRegNum()); if (op->OperIs(GT_COPY)) { GenTree* srcOp = op->gtGetOp1(); - consumedRegs |= genRegMask(srcOp->GetRegNum()); + consumedRegs |= genSingleTypeRegMask(srcOp->GetRegNum()); } else if (op->IsLocal()) { @@ -12830,7 +12853,7 @@ void LinearScan::RegisterSelection::try_BEST_FIT() for (SingleTypeRegSet bestFitCandidates = candidates; bestFitCandidates != RBM_NONE;) { regNumber bestFitCandidateRegNum = genFirstRegNumFromMask(bestFitCandidates); - SingleTypeRegSet bestFitCandidateBit = genRegMask(bestFitCandidateRegNum); + SingleTypeRegSet bestFitCandidateBit = genSingleTypeRegMask(bestFitCandidateRegNum); bestFitCandidates ^= bestFitCandidateBit; // Find the next RefPosition of the register. @@ -12929,7 +12952,7 @@ void LinearScan::RegisterSelection::try_REG_ORDER() for (SingleTypeRegSet regOrderCandidates = candidates; regOrderCandidates != RBM_NONE;) { regNumber regOrderCandidateRegNum = genFirstRegNumFromMask(regOrderCandidates); - SingleTypeRegSet regOrderCandidateBit = genRegMask(regOrderCandidateRegNum); + SingleTypeRegSet regOrderCandidateBit = genSingleTypeRegMask(regOrderCandidateRegNum); regOrderCandidates ^= regOrderCandidateBit; unsigned thisRegOrder = linearScan->getRegisterRecord(regOrderCandidateRegNum)->regOrder; @@ -12965,7 +12988,7 @@ void LinearScan::RegisterSelection::try_SPILL_COST() for (SingleTypeRegSet spillCandidates = candidates; spillCandidates != RBM_NONE;) { regNumber spillCandidateRegNum = genFirstRegNumFromMask(spillCandidates); - SingleTypeRegSet spillCandidateBit = genRegMask(spillCandidateRegNum); + SingleTypeRegSet spillCandidateBit = genSingleTypeRegMask(spillCandidateRegNum); spillCandidates ^= spillCandidateBit; RegRecord* spillCandidateRegRecord = &linearScan->physRegs[spillCandidateRegNum]; @@ -13090,7 +13113,7 @@ void LinearScan::RegisterSelection::try_FAR_NEXT_REF() for (SingleTypeRegSet farthestCandidates = candidates; farthestCandidates != RBM_NONE;) { regNumber farthestCandidateRegNum = genFirstRegNumFromMask(farthestCandidates); - SingleTypeRegSet farthestCandidateBit = genRegMask(farthestCandidateRegNum); + SingleTypeRegSet farthestCandidateBit = genSingleTypeRegMask(farthestCandidateRegNum); farthestCandidates ^= farthestCandidateBit; // Find the next RefPosition of the register. @@ -13123,7 +13146,7 @@ void LinearScan::RegisterSelection::try_PREV_REG_OPT() for (SingleTypeRegSet prevRegOptCandidates = candidates; prevRegOptCandidates != RBM_NONE;) { regNumber prevRegOptCandidateRegNum = genFirstRegNumFromMask(prevRegOptCandidates); - SingleTypeRegSet prevRegOptCandidateBit = genRegMask(prevRegOptCandidateRegNum); + SingleTypeRegSet prevRegOptCandidateBit = genSingleTypeRegMask(prevRegOptCandidateRegNum); prevRegOptCandidates ^= prevRegOptCandidateBit; Interval* assignedInterval = linearScan->physRegs[prevRegOptCandidateRegNum].assignedInterval; bool foundPrevRegOptReg = true; @@ -13226,7 +13249,7 @@ void LinearScan::RegisterSelection::calculateUnassignedSets() // TODO: Seperate for (; coversCandidates != RBM_NONE;) { regNumber coversCandidateRegNum = genFirstRegNumFromMask(coversCandidates); - SingleTypeRegSet coversCandidateBit = genRegMask(coversCandidateRegNum); + SingleTypeRegSet coversCandidateBit = genSingleTypeRegMask(coversCandidateRegNum); coversCandidates ^= coversCandidateBit; // The register is considered unassigned if it has no assignedInterval, OR @@ -13254,7 +13277,7 @@ void LinearScan::RegisterSelection::calculateCoversSets() for (; coversCandidates != RBM_NONE;) { regNumber coversCandidateRegNum = genFirstRegNumFromMask(coversCandidates); - SingleTypeRegSet coversCandidateBit = genRegMask(coversCandidateRegNum); + SingleTypeRegSet coversCandidateBit = genSingleTypeRegMask(coversCandidateRegNum); coversCandidates ^= coversCandidateBit; // If we have a single candidate we don't need to compute the preference-related sets, but we @@ -13565,7 +13588,7 @@ SingleTypeRegSet LinearScan::RegisterSelection::select(Interval* while (checkConflictMask != RBM_NONE) { regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask); - SingleTypeRegSet checkConflictBit = genRegMask(checkConflictReg); + SingleTypeRegSet checkConflictBit = genSingleTypeRegMask(checkConflictReg); checkConflictMask ^= checkConflictBit; LsraLocation checkConflictLocation = linearScan->nextFixedRef[checkConflictReg]; @@ -13590,7 +13613,7 @@ SingleTypeRegSet LinearScan::RegisterSelection::select(Interval* if (!found && (currentInterval->assignedReg != nullptr)) { RegRecord* prevRegRec = currentInterval->assignedReg; - prevRegBit = genRegMask(prevRegRec->regNum); + prevRegBit = genSingleTypeRegMask(prevRegRec->regNum); if ((prevRegRec->assignedInterval == currentInterval) && ((candidates & prevRegBit) != RBM_NONE)) { if (!needsConsecutiveRegisters) @@ -13884,7 +13907,7 @@ SingleTypeRegSet LinearScan::RegisterSelection::selectMinimal( while (checkConflictMask != RBM_NONE) { regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask); - SingleTypeRegSet checkConflictBit = genRegMask(checkConflictReg); + SingleTypeRegSet checkConflictBit = genSingleTypeRegMask(checkConflictReg); checkConflictMask ^= checkConflictBit; LsraLocation checkConflictLocation = linearScan->nextFixedRef[checkConflictReg]; diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index e20a92a695426..4ad3bbc7f840c 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -2097,28 +2097,28 @@ class LinearScan : public LinearScanInterface int BuildLclHeap(GenTree* tree); #if defined(TARGET_AMD64) - SingleTypeRegSet rbmAllFloat; - SingleTypeRegSet rbmFltCalleeTrash; + regMaskTP rbmAllFloat; + regMaskTP rbmFltCalleeTrash; - FORCEINLINE SingleTypeRegSet get_RBM_ALLFLOAT() const + FORCEINLINE regMaskTP get_RBM_ALLFLOAT() const { return this->rbmAllFloat; } - FORCEINLINE SingleTypeRegSet get_RBM_FLT_CALLEE_TRASH() const + FORCEINLINE regMaskTP get_RBM_FLT_CALLEE_TRASH() const { return this->rbmFltCalleeTrash; } #endif // TARGET_AMD64 #if defined(TARGET_XARCH) - SingleTypeRegSet rbmAllMask; - SingleTypeRegSet rbmMskCalleeTrash; + regMaskTP rbmAllMask; + regMaskTP rbmMskCalleeTrash; - FORCEINLINE SingleTypeRegSet get_RBM_ALLMASK() const + FORCEINLINE regMaskTP get_RBM_ALLMASK() const { return this->rbmAllMask; } - FORCEINLINE SingleTypeRegSet get_RBM_MSK_CALLEE_TRASH() const + FORCEINLINE regMaskTP get_RBM_MSK_CALLEE_TRASH() const { return this->rbmMskCalleeTrash; } @@ -2139,21 +2139,21 @@ class LinearScan : public LinearScanInterface // static FORCEINLINE SingleTypeRegSet calleeSaveRegs(RegisterType rt) { - static const SingleTypeRegSet varTypeCalleeSaveRegs[] = { + static const regMaskTP varTypeCalleeSaveRegs[] = { #define DEF_TP(tn, nm, jitType, sz, sze, asze, st, al, regTyp, regFld, csr, ctr, tf) csr, #include "typelist.h" #undef DEF_TP }; assert((unsigned)rt < ArrLen(varTypeCalleeSaveRegs)); - return varTypeCalleeSaveRegs[rt]; + return varTypeCalleeSaveRegs[rt].GetRegSetForType(rt); } #if defined(TARGET_XARCH) // Not all of the callee trash values are constant, so don't declare this as a method local static // doing so results in significantly more complex codegen and we'd rather just initialize this once // as part of initializing LSRA instead - SingleTypeRegSet varTypeCalleeTrashRegs[TYP_COUNT]; + regMaskTP varTypeCalleeTrashRegs[TYP_COUNT]; #endif // TARGET_XARCH //------------------------------------------------------------------------ @@ -2162,7 +2162,7 @@ class LinearScan : public LinearScanInterface FORCEINLINE SingleTypeRegSet callerSaveRegs(RegisterType rt) const { #if !defined(TARGET_XARCH) - static const SingleTypeRegSet varTypeCalleeTrashRegs[] = { + static const regMaskTP varTypeCalleeTrashRegs[] = { #define DEF_TP(tn, nm, jitType, sz, sze, asze, st, al, regTyp, regFld, csr, ctr, tf) ctr, #include "typelist.h" #undef DEF_TP @@ -2170,7 +2170,7 @@ class LinearScan : public LinearScanInterface #endif // !TARGET_XARCH assert((unsigned)rt < ArrLen(varTypeCalleeTrashRegs)); - return varTypeCalleeTrashRegs[rt]; + return varTypeCalleeTrashRegs[rt].GetRegSetForType(rt); } }; @@ -2338,7 +2338,8 @@ class Interval : public Referenceable { // This uses regMasks to handle the case where a double actually occupies two registers // TODO-Throughput: This could/should be done more cheaply. - return (physReg != REG_NA && (genRegMask(physReg, registerType) & genRegMask(regNum)) != RBM_NONE); + return (physReg != REG_NA && + (genSingleTypeRegMask(physReg, registerType) & genSingleTypeRegMask(regNum)) != RBM_NONE); } // Assign the related interval. @@ -2387,7 +2388,7 @@ class Interval : public Referenceable // SingleTypeRegSet getCurrentPreferences() { - return (assignedReg == nullptr) ? registerPreferences : genRegMask(assignedReg->regNum); + return (assignedReg == nullptr) ? registerPreferences : genSingleTypeRegMask(assignedReg->regNum); } void mergeRegisterPreferences(SingleTypeRegSet preferences) @@ -2659,7 +2660,7 @@ class RefPosition { referent = r; isPhysRegRef = true; - registerAssignment = genRegMask(r->regNum); + registerAssignment = genSingleTypeRegMask(r->regNum); } regNumber assignedReg() diff --git a/src/coreclr/jit/lsraarm.cpp b/src/coreclr/jit/lsraarm.cpp index f2c60cde13eb0..fc77279eabb75 100644 --- a/src/coreclr/jit/lsraarm.cpp +++ b/src/coreclr/jit/lsraarm.cpp @@ -670,7 +670,7 @@ int LinearScan::BuildNode(GenTree* tree) SingleTypeRegSet argMask = RBM_NONE; if (argReg != REG_COUNT) { - argMask = genRegMask(argReg); + argMask = genSingleTypeRegMask(argReg); } // If type of node is `long` then it is actually `double`. @@ -679,7 +679,7 @@ int LinearScan::BuildNode(GenTree* tree) { dstCount++; assert(genRegArgNext(argReg) == REG_NEXT(argReg)); - argMask |= genRegMask(REG_NEXT(argReg)); + argMask |= genSingleTypeRegMask(REG_NEXT(argReg)); dstCount = 2; } if (!tree->gtGetOp1()->isContained()) diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 7a35c36e774d9..6ea2f6e5f97ff 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -103,7 +103,7 @@ void LinearScan::assignConsecutiveRegisters(RefPosition* firstRefPosition, regNu #endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE INDEBUG(refPosCount++); assert((consecutiveRefPosition->refType == RefTypeDef) || (consecutiveRefPosition->refType == RefTypeUse)); - consecutiveRefPosition->registerAssignment = genRegMask(regToAssign); + consecutiveRefPosition->registerAssignment = genSingleTypeRegMask(regToAssign); consecutiveRefPosition = getNextConsecutiveRefPosition(consecutiveRefPosition); regToAssign = regToAssign == REG_FP_LAST ? REG_FP_FIRST : REG_NEXT(regToAssign); } diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index 85f1f66442404..323dea8d4809a 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -174,7 +174,8 @@ int LinearScan::BuildCall(GenTreeCall* call) ctrlExprCandidates = allRegs(TYP_INT) & RBM_INT_CALLEE_TRASH & ~RBM_LR; if (compiler->getNeedsGSSecurityCookie()) { - ctrlExprCandidates &= ~(genRegMask(REG_GSCOOKIE_TMP_0) | genRegMask(REG_GSCOOKIE_TMP_1)); + ctrlExprCandidates &= + ~(genSingleTypeRegMask(REG_GSCOOKIE_TMP_0) | genSingleTypeRegMask(REG_GSCOOKIE_TMP_1)); } assert(ctrlExprCandidates != RBM_NONE); } @@ -291,7 +292,7 @@ int LinearScan::BuildCall(GenTreeCall* call) } #endif // TARGET_ARM #endif - BuildUse(use.GetNode(), genRegMask(use.GetNode()->GetRegNum())); + BuildUse(use.GetNode(), genSingleTypeRegMask(use.GetNode()->GetRegNum())); srcCount++; } } @@ -301,7 +302,7 @@ int LinearScan::BuildCall(GenTreeCall* call) assert(regCount == abiInfo.NumRegs); for (unsigned int i = 0; i < regCount; i++) { - BuildUse(argNode, genRegMask(argNode->AsPutArgSplit()->GetRegNumByIdx(i)), i); + BuildUse(argNode, genSingleTypeRegMask(argNode->AsPutArgSplit()->GetRegNumByIdx(i)), i); } srcCount += regCount; } @@ -317,14 +318,14 @@ int LinearScan::BuildCall(GenTreeCall* call) if (argNode->TypeGet() == TYP_LONG) { assert(argNode->IsMultiRegNode()); - BuildUse(argNode, genRegMask(argNode->GetRegNum()), 0); - BuildUse(argNode, genRegMask(genRegArgNext(argNode->GetRegNum())), 1); + BuildUse(argNode, genSingleTypeRegMask(argNode->GetRegNum()), 0); + BuildUse(argNode, genSingleTypeRegMask(genRegArgNext(argNode->GetRegNum())), 1); srcCount += 2; } else #endif // TARGET_ARM { - BuildUse(argNode, genRegMask(argNode->GetRegNum())); + BuildUse(argNode, genSingleTypeRegMask(argNode->GetRegNum())); srcCount++; } } @@ -384,9 +385,9 @@ int LinearScan::BuildCall(GenTreeCall* call) // that we will attach to this node to guarantee that they are available // during generating this node. assert(call->gtFlags & GTF_TLS_GET_ADDR); - newRefPosition(REG_R0, currentLoc, RefTypeFixedReg, nullptr, genRegMask(REG_R0)); - newRefPosition(REG_R1, currentLoc, RefTypeFixedReg, nullptr, genRegMask(REG_R1)); - ctrlExprCandidates = genRegMask(REG_R2); + newRefPosition(REG_R0, currentLoc, RefTypeFixedReg, nullptr, genSingleTypeRegMask(REG_R0)); + newRefPosition(REG_R1, currentLoc, RefTypeFixedReg, nullptr, genSingleTypeRegMask(REG_R1)); + ctrlExprCandidates = genSingleTypeRegMask(REG_R2); } #endif BuildUse(ctrlExpr, ctrlExprCandidates); @@ -541,7 +542,7 @@ int LinearScan::BuildPutArgSplit(GenTreePutArgSplit* argNode) for (unsigned i = 0; i < argNode->gtNumRegs; i++) { regNumber thisArgReg = (regNumber)((unsigned)argReg + i); - argMask |= genRegMask(thisArgReg); + argMask |= genSingleTypeRegMask(thisArgReg); argNode->SetRegNumByIdx(thisArgReg, i); } assert((argMask == RBM_NONE) || ((argMask & availableIntRegs) != RBM_NONE) || @@ -582,7 +583,7 @@ int LinearScan::BuildPutArgSplit(GenTreePutArgSplit* argNode) SingleTypeRegSet sourceMask = RBM_NONE; if (sourceRegCount < argNode->gtNumRegs) { - sourceMask = genRegMask((regNumber)((unsigned)argReg + sourceRegCount)); + sourceMask = genSingleTypeRegMask((regNumber)((unsigned)argReg + sourceRegCount)); } sourceRegCount++; BuildUse(node, sourceMask, regIndex); diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 90dd50336d655..bd15db5908341 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -2870,7 +2870,7 @@ void LinearScan::buildInitialParamDef(const LclVarDsc* varDsc, regNumber paramRe { // Set this interval as currently assigned to that register assert(paramReg < REG_COUNT); - mask = genRegMask(paramReg); + mask = genSingleTypeRegMask(paramReg); assignPhysReg(paramReg, interval); INDEBUG(registersToDump |= getRegMask(paramReg, interval->registerType)); } @@ -2933,7 +2933,7 @@ void LinearScan::stressSetRandomParameterPreferences() } *regs &= ~genRegMask(prefReg); - interval->mergeRegisterPreferences(genRegMask(prefReg)); + interval->mergeRegisterPreferences(genSingleTypeRegMask(prefReg)); } } @@ -3085,7 +3085,7 @@ RefPosition* LinearScan::BuildDef(GenTree* tree, SingleTypeRegSet dstCandidates, if (!tree->IsMultiRegNode() || (multiRegIdx == 0)) { assert((dstCandidates == RBM_NONE) || (dstCandidates == genRegMask(tree->GetRegNum()))); - dstCandidates = genRegMask(tree->GetRegNum()); + dstCandidates = genSingleTypeRegMask(tree->GetRegNum()); } else { @@ -3166,7 +3166,7 @@ void LinearScan::BuildCallDefs(GenTree* tree, int dstCount, regMaskTP dstCandida assert(dstCandidates.IsRegNumInMask(thisReg)); dstCandidates.RemoveRegNumFromMask(thisReg); - BuildDef(tree, genRegMask(thisReg), i); + BuildDef(tree, genSingleTypeRegMask(thisReg), i); } } @@ -4225,7 +4225,7 @@ int LinearScan::BuildReturn(GenTree* tree) { hasMismatchedRegTypes = true; SingleTypeRegSet dstRegMask = - genRegMask(retTypeDesc.GetABIReturnReg(i, compiler->info.compCallConv)); + genSingleTypeRegMask(retTypeDesc.GetABIReturnReg(i, compiler->info.compCallConv)); if (varTypeUsesIntReg(dstType)) { @@ -4252,7 +4252,9 @@ int LinearScan::BuildReturn(GenTree* tree) if (!hasMismatchedRegTypes || (regType(op1->AsLclVar()->GetFieldTypeByIndex(compiler, i)) == regType(retTypeDesc.GetReturnRegType(i)))) { - BuildUse(op1, genRegMask(retTypeDesc.GetABIReturnReg(i, compiler->info.compCallConv)), i); + BuildUse(op1, + genSingleTypeRegMask(retTypeDesc.GetABIReturnReg(i, compiler->info.compCallConv)), + i); } else { @@ -4280,7 +4282,11 @@ int LinearScan::BuildReturn(GenTree* tree) break; case TYP_DOUBLE: // We ONLY want the valid double register in the RBM_DOUBLERET mask. - useCandidates = (RBM_DOUBLERET & RBM_ALLDOUBLE); +#ifdef TARGET_AMD64 + useCandidates = (RBM_DOUBLERET & RBM_ALLDOUBLE).GetFloatRegSet(); +#else + useCandidates = (RBM_DOUBLERET & RBM_ALLDOUBLE); +#endif // TARGET_AMD64 break; case TYP_LONG: useCandidates = RBM_LNGRET; @@ -4379,7 +4385,7 @@ int LinearScan::BuildPutArgReg(GenTreeUnOp* node) // To avoid redundant moves, have the argument operand computed in the // register in which the argument is passed to the call. - SingleTypeRegSet argMask = genRegMask(argReg); + SingleTypeRegSet argMask = genSingleTypeRegMask(argReg); RefPosition* use = BuildUse(op1, argMask); // Record that this register is occupied by a register now. @@ -4411,7 +4417,7 @@ int LinearScan::BuildPutArgReg(GenTreeUnOp* node) if (node->TypeGet() == TYP_LONG) { srcCount++; - SingleTypeRegSet argMaskHi = genRegMask(REG_NEXT(argReg)); + SingleTypeRegSet argMaskHi = genSingleTypeRegMask(REG_NEXT(argReg)); assert(genRegArgNext(argReg) == REG_NEXT(argReg)); use = BuildUse(op1, argMaskHi, 1); BuildDef(node, argMask, 0); @@ -4457,7 +4463,7 @@ void LinearScan::HandleFloatVarArgs(GenTreeCall* call, GenTree* argNode, bool* c regNumber argReg = argNode->GetRegNum(); regNumber targetReg = compiler->getCallArgIntRegister(argReg); - buildInternalIntRegisterDefForNode(call, genRegMask(targetReg)); + buildInternalIntRegisterDefForNode(call, genSingleTypeRegMask(targetReg)); } } diff --git a/src/coreclr/jit/lsraloongarch64.cpp b/src/coreclr/jit/lsraloongarch64.cpp index 560171b86ea94..dd2805be4bcbe 100644 --- a/src/coreclr/jit/lsraloongarch64.cpp +++ b/src/coreclr/jit/lsraloongarch64.cpp @@ -794,7 +794,7 @@ int LinearScan::BuildCall(GenTreeCall* call) #ifdef DEBUG assert(use.GetNode()->OperIs(GT_PUTARG_REG)); #endif - BuildUse(use.GetNode(), genRegMask(use.GetNode()->GetRegNum())); + BuildUse(use.GetNode(), genSingleTypeRegMask(use.GetNode()->GetRegNum())); srcCount++; } } @@ -804,7 +804,7 @@ int LinearScan::BuildCall(GenTreeCall* call) assert(regCount == abiInfo.NumRegs); for (unsigned int i = 0; i < regCount; i++) { - BuildUse(argNode, genRegMask(argNode->AsPutArgSplit()->GetRegNumByIdx(i)), i); + BuildUse(argNode, genSingleTypeRegMask(argNode->AsPutArgSplit()->GetRegNumByIdx(i)), i); } srcCount += regCount; } @@ -814,7 +814,7 @@ int LinearScan::BuildCall(GenTreeCall* call) assert(argNode->GetRegNum() == argReg); HandleFloatVarArgs(call, argNode, &callHasFloatRegArgs); { - BuildUse(argNode, genRegMask(argNode->GetRegNum())); + BuildUse(argNode, genSingleTypeRegMask(argNode->GetRegNum())); srcCount++; } } @@ -999,7 +999,7 @@ int LinearScan::BuildPutArgSplit(GenTreePutArgSplit* argNode) for (unsigned i = 0; i < argNode->gtNumRegs; i++) { regNumber thisArgReg = (regNumber)((unsigned)argReg + i); - argMask |= genRegMask(thisArgReg); + argMask |= genSingleTypeRegMask(thisArgReg); argNode->SetRegNumByIdx(thisArgReg, i); } assert((argMask == RBM_NONE) || ((argMask & availableIntRegs) != RBM_NONE) || @@ -1028,7 +1028,7 @@ int LinearScan::BuildPutArgSplit(GenTreePutArgSplit* argNode) SingleTypeRegSet sourceMask = RBM_NONE; if (sourceRegCount < argNode->gtNumRegs) { - sourceMask = genRegMask((regNumber)((unsigned)argReg + sourceRegCount)); + sourceMask = genSingleTypeRegMask((regNumber)((unsigned)argReg + sourceRegCount)); } sourceRegCount++; BuildUse(node, sourceMask, 0); diff --git a/src/coreclr/jit/lsrariscv64.cpp b/src/coreclr/jit/lsrariscv64.cpp index ebea9cce71472..2983c035fe8d1 100644 --- a/src/coreclr/jit/lsrariscv64.cpp +++ b/src/coreclr/jit/lsrariscv64.cpp @@ -885,7 +885,8 @@ int LinearScan::BuildCall(GenTreeCall* call) ctrlExprCandidates = allRegs(TYP_INT) & RBM_INT_CALLEE_TRASH; if (compiler->getNeedsGSSecurityCookie()) { - ctrlExprCandidates &= ~(genRegMask(REG_GSCOOKIE_TMP_0) | genRegMask(REG_GSCOOKIE_TMP_1)); + ctrlExprCandidates &= + ~(genSingleTypeRegMask(REG_GSCOOKIE_TMP_0) | genSingleTypeRegMask(REG_GSCOOKIE_TMP_1)); } assert(ctrlExprCandidates != RBM_NONE); } @@ -957,7 +958,7 @@ int LinearScan::BuildCall(GenTreeCall* call) #ifdef DEBUG assert(use.GetNode()->OperIs(GT_PUTARG_REG)); #endif - BuildUse(use.GetNode(), genRegMask(use.GetNode()->GetRegNum())); + BuildUse(use.GetNode(), genSingleTypeRegMask(use.GetNode()->GetRegNum())); srcCount++; } } @@ -967,7 +968,7 @@ int LinearScan::BuildCall(GenTreeCall* call) assert(regCount == abiInfo.NumRegs); for (unsigned int i = 0; i < regCount; i++) { - BuildUse(argNode, genRegMask(argNode->AsPutArgSplit()->GetRegNumByIdx(i)), i); + BuildUse(argNode, genSingleTypeRegMask(argNode->AsPutArgSplit()->GetRegNumByIdx(i)), i); } srcCount += regCount; } @@ -977,7 +978,7 @@ int LinearScan::BuildCall(GenTreeCall* call) assert(argNode->GetRegNum() == argReg); HandleFloatVarArgs(call, argNode, &callHasFloatRegArgs); { - BuildUse(argNode, genRegMask(argNode->GetRegNum())); + BuildUse(argNode, genSingleTypeRegMask(argNode->GetRegNum())); srcCount++; } } @@ -1149,7 +1150,7 @@ int LinearScan::BuildPutArgSplit(GenTreePutArgSplit* argNode) for (unsigned i = 0; i < argNode->gtNumRegs; i++) { regNumber thisArgReg = (regNumber)((unsigned)argReg + i); - argMask |= genRegMask(thisArgReg); + argMask |= genSingleTypeRegMask(thisArgReg); argNode->SetRegNumByIdx(thisArgReg, i); } assert((argMask == RBM_NONE) || ((argMask & availableIntRegs) != RBM_NONE) || @@ -1181,7 +1182,7 @@ int LinearScan::BuildPutArgSplit(GenTreePutArgSplit* argNode) SingleTypeRegSet sourceMask = RBM_NONE; if (sourceRegCount < argNode->gtNumRegs) { - sourceMask = genRegMask((regNumber)((unsigned)argReg + sourceRegCount)); + sourceMask = genSingleTypeRegMask((regNumber)((unsigned)argReg + sourceRegCount)); } sourceRegCount++; BuildUse(node, sourceMask, regIndex); diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index fe21be0ec8938..d3a7f075fdd08 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -1259,7 +1259,7 @@ int LinearScan::BuildCall(GenTreeCall* call) if (argNode->OperIsPutArgReg()) { srcCount++; - BuildUse(argNode, genRegMask(argNode->GetRegNum())); + BuildUse(argNode, genSingleTypeRegMask(argNode->GetRegNum())); } #ifdef UNIX_AMD64_ABI else if (argNode->OperGet() == GT_FIELD_LIST) @@ -1268,7 +1268,7 @@ int LinearScan::BuildCall(GenTreeCall* call) { assert(use.GetNode()->OperIsPutArgReg()); srcCount++; - BuildUse(use.GetNode(), genRegMask(use.GetNode()->GetRegNum())); + BuildUse(use.GetNode(), genSingleTypeRegMask(use.GetNode()->GetRegNum())); } } #endif // UNIX_AMD64_ABI diff --git a/src/coreclr/jit/regMaskTPOps.cpp b/src/coreclr/jit/regMaskTPOps.cpp index 86de50a08cb95..30654e34a34e8 100644 --- a/src/coreclr/jit/regMaskTPOps.cpp +++ b/src/coreclr/jit/regMaskTPOps.cpp @@ -13,7 +13,7 @@ struct regMaskTP; // void regMaskTP::RemoveRegNumFromMask(regNumber reg) { - low &= ~genRegMask(reg); + low &= ~genSingleTypeRegMask(reg); } //------------------------------------------------------------------------ diff --git a/src/coreclr/jit/target.h b/src/coreclr/jit/target.h index 7e144eb9c5b21..932c4ca41cabb 100644 --- a/src/coreclr/jit/target.h +++ b/src/coreclr/jit/target.h @@ -293,6 +293,21 @@ struct regMaskTP return getLow(); } + SingleTypeRegSet GetIntRegSet() const + { + return getLow(); + } + + SingleTypeRegSet GetFloatRegSet() const + { + return getLow(); + } + + SingleTypeRegSet GetPredicateRegSet() const + { + return getLow(); + } + void RemoveRegNumFromMask(regNumber reg); bool IsRegNumInMask(regNumber reg); @@ -508,8 +523,8 @@ inline bool isByteReg(regNumber reg) } #endif -inline SingleTypeRegSet genRegMask(regNumber reg); -inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE)); +inline regMaskTP genRegMask(regNumber reg); +inline regMaskTP genRegMaskFloat(regNumber reg ARM_ARG(var_types type = TYP_DOUBLE)); /***************************************************************************** * Return true if the register number is valid @@ -732,35 +747,13 @@ inline bool floatRegCanHoldType(regNumber reg, var_types type) } #endif -/***************************************************************************** - * - * Map a register number to a register mask. - */ - extern const regMaskSmall regMasks[REG_COUNT]; -inline SingleTypeRegSet genRegMask(regNumber reg) -{ - assert((unsigned)reg < ArrLen(regMasks)); -#ifdef TARGET_AMD64 - // shift is faster than a L1 hit on modern x86 - // (L1 latency on sandy bridge is 4 cycles for [base] and 5 for [base + index*c] ) - // the reason this is AMD-only is because the x86 BE will try to get reg masks for REG_STK - // and the result needs to be zero. - SingleTypeRegSet result = 1ULL << reg; - assert(result == regMasks[reg]); - return result; -#else - return regMasks[reg]; -#endif -} - /***************************************************************************** * * Map a register number to a floating-point register mask. */ - -inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type /* = TYP_DOUBLE */)) +inline SingleTypeRegSet genSingleTypeFloatMask(regNumber reg ARM_ARG(var_types type /* = TYP_DOUBLE */)) { #if defined(TARGET_AMD64) || defined(TARGET_ARM64) || defined(TARGET_X86) || defined(TARGET_LOONGARCH64) || \ defined(TARGET_RISCV64) @@ -785,45 +778,112 @@ inline SingleTypeRegSet genRegMaskFloat(regNumber reg ARM_ARG(var_types type /* } //------------------------------------------------------------------------ -// genRegMask: Given a register, and its type, generate the appropriate regMask +// genSingleTypeRegMask: Given a register, generate the appropriate regMask +// +// Arguments: +// regNum - the register of interest +// +// Return Value: +// This will usually return the same value as genRegMask(regNum), except +// that it will return a 64-bits (or 32-bits) entity instead of `regMaskTP`. +// +inline SingleTypeRegSet genSingleTypeRegMask(regNumber reg) +{ + assert((unsigned)reg < ArrLen(regMasks)); +#ifdef TARGET_AMD64 + // shift is faster than a L1 hit on modern x86 + // (L1 latency on sandy bridge is 4 cycles for [base] and 5 for [base + index*c] ) + // the reason this is AMD-only is because the x86 BE will try to get reg masks for REG_STK + // and the result needs to be zero. + SingleTypeRegSet result = 1ULL << reg; + assert(result == regMasks[reg]); + return result; +#else + return regMasks[reg]; +#endif +} + +//------------------------------------------------------------------------ +// genSingleTypeRegMask: Given a register, generate the appropriate regMask // // Arguments: // regNum - the register of interest // type - the type of regNum (i.e. the type it is being used as) // // Return Value: -// This will usually return the same value as genRegMask(regNum), but -// on architectures where multiple registers are used for certain types +// This will usually return the same value as genRegMask(regNum), except +// that it will return a 64-bits (or 32-bits) entity instead of `regMaskTP`. +// On architectures where multiple registers are used for certain types // (e.g. TYP_DOUBLE on ARM), it will return a regMask that includes -// all the registers. -// Registers that are used in pairs, but separately named (e.g. TYP_LONG -// on ARM) will return just the regMask for the given register. -// -// Assumptions: -// For registers that are used in pairs, the caller will be handling -// each member of the pair separately. +// all the registers for that type. // -inline SingleTypeRegSet genRegMask(regNumber regNum, var_types type) +inline SingleTypeRegSet genSingleTypeRegMask(regNumber regNum, var_types type) { #if defined(TARGET_ARM) SingleTypeRegSet regMask = RBM_NONE; if (varTypeUsesIntReg(type)) { - regMask = genRegMask(regNum); + regMask = genSingleTypeRegMask(regNum); } else { assert(varTypeUsesFloatReg(type)); - regMask = genRegMaskFloat(regNum, type); + regMask = genSingleTypeFloatMask(regNum, type); } return regMask; #else - return genRegMask(regNum); + return genSingleTypeRegMask(regNum); #endif } +/***************************************************************************** + * + * Map a register number to a register mask. + */ + +inline regMaskTP genRegMask(regNumber reg) +{ + // TODO: Populate regMaskTP based on reg + return genSingleTypeRegMask(reg); +} + +/***************************************************************************** + * + * Map a register number to a floating-point register mask. + */ + +inline regMaskTP genRegMaskFloat(regNumber reg ARM_ARG(var_types type /* = TYP_DOUBLE */)) +{ + return regMaskTP(genSingleTypeFloatMask(reg ARM_ARG(type))); +} + +//------------------------------------------------------------------------ +// genRegMask: Given a register, and its type, generate the appropriate regMask +// +// Arguments: +// regNum - the register of interest +// type - the type of regNum (i.e. the type it is being used as) +// +// Return Value: +// This will usually return the same value as genRegMask(regNum), but +// on architectures where multiple registers are used for certain types +// (e.g. TYP_DOUBLE on ARM), it will return a regMask that includes +// all the registers. +// Registers that are used in pairs, but separately named (e.g. TYP_LONG +// on ARM) will return just the regMask for the given register. +// +// Assumptions: +// For registers that are used in pairs, the caller will be handling +// each member of the pair separately. +// +inline regMaskTP genRegMask(regNumber regNum, var_types type) +{ + // TODO: Populate regMaskTP based on regNum/type + return genSingleTypeRegMask(regNum ARM_ARG(type)); +} + /***************************************************************************** * * These arrays list the callee-saved register numbers (and bitmaps, respectively) for