From 9aba34270549573e49fcc60c5c1bc78ba7342718 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 30 Jul 2025 14:20:22 +0800 Subject: [PATCH 01/12] Precommit tests --- llvm/test/CodeGen/RISCV/rvv/vl-opt.ll | 53 +++++++++++++++++++ llvm/test/CodeGen/RISCV/rvv/vl-opt.mir | 71 +++++++++++++++++++++++++- 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll b/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll index cd282c265ae47..8651eaad4ef28 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll @@ -198,3 +198,56 @@ define void @fadd_fcmp_select_copy( %v, %c call void @llvm.riscv.vsm( %select, ptr %p, iXLen %vl) ret void } + +define void @recurrence( %v, ptr %p, iXLen %n, iXLen %vl) { +; CHECK-LABEL: recurrence: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vsetvli a3, zero, e32, m2, ta, ma +; CHECK-NEXT: vmv.v.i v10, 0 +; CHECK-NEXT: .LBB13_1: # %loop +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: addi a1, a1, -1 +; CHECK-NEXT: vadd.vv v10, v10, v8 +; CHECK-NEXT: bnez a1, .LBB13_1 +; CHECK-NEXT: # %bb.2: # %exit +; CHECK-NEXT: vsetvli zero, a2, e32, m2, ta, ma +; CHECK-NEXT: vse32.v v10, (a0) +; CHECK-NEXT: ret +entry: + br label %loop +loop: + %iv = phi iXLen [ 0, %entry ], [ %iv.next, %loop ] + %phi = phi [ zeroinitializer, %entry ], [ %x, %loop ] + %x = add %phi, %v + %iv.next = add iXLen %iv, 1 + %done = icmp eq iXLen %iv.next, %n + br i1 %done, label %exit, label %loop +exit: + call void @llvm.riscv.vse( %x, ptr %p, iXLen %vl) + ret void +} + +define @phi( %v, i1 %cond, iXLen %vl) { +; CHECK-LABEL: phi: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: andi a0, a0, 1 +; CHECK-NEXT: vsetivli zero, 2, e32, m2, ta, ma +; CHECK-NEXT: vadd.vi v8, v8, 1 +; CHECK-NEXT: beqz a0, .LBB14_2 +; CHECK-NEXT: # %bb.1: # %foo +; CHECK-NEXT: vsetivli zero, 1, e32, m2, ta, ma +; CHECK-NEXT: vadd.vi v8, v8, 1 +; CHECK-NEXT: ret +; CHECK-NEXT: .LBB14_2: # %bar +; CHECK-NEXT: vadd.vi v8, v8, 2 +; CHECK-NEXT: ret +entry: + %a = call @llvm.riscv.vadd( poison, %v, iXLen 1, iXLen -1) + br i1 %cond, label %foo, label %bar +foo: + %b = call @llvm.riscv.vadd( poison, %a, iXLen 1, iXLen 1) + ret %b +bar: + %c = call @llvm.riscv.vadd( poison, %a, iXLen 2, iXLen 2) + ret %c +} diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt.mir b/llvm/test/CodeGen/RISCV/rvv/vl-opt.mir index 60398cdf1db66..538b68a70908b 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vl-opt.mir +++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt.mir @@ -603,4 +603,73 @@ body: | $x10 = COPY %9 PseudoRET implicit $x10 ... - +--- +name: recurrence +tracksRegLiveness: true +body: | + ; CHECK-LABEL: name: recurrence + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: liveins: $x8 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %avl:gprnox0 = COPY $x8 + ; CHECK-NEXT: %start:vr = PseudoVMV_V_I_M1 $noreg, 0, -1, 3 /* e8 */, 3 /* ta, ma */ + ; CHECK-NEXT: PseudoBR %bb.1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %phi:vr = PHI %start, %bb.0, %inc, %bb.1 + ; CHECK-NEXT: %inc:vr = PseudoVADD_VI_M1 $noreg, %phi, 1, -1, 3 /* e8 */, 3 /* ta, ma */ + ; CHECK-NEXT: BNE $noreg, $noreg, %bb.1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: PseudoVSE8_V_M1 %inc, $noreg, %avl, 3 /* e8 */ + bb.0: + liveins: $x8 + %avl:gprnox0 = COPY $x8 + %start:vr = PseudoVMV_V_I_M1 $noreg, 0, -1, 3 /* e8 */, 3, /* ta, ma */ + PseudoBR %bb.1 + bb.1: + %phi:vr = PHI %start, %bb.0, %inc, %bb.1 + %inc:vr = PseudoVADD_VI_M1 $noreg, %phi, 1, -1, 3 /* e8 */, 3 /* ta, ma */ + BNE $noreg, $noreg, %bb.1 + bb.2: + PseudoVSE8_V_M1 %inc, $noreg, %avl, 3 /* e8 */ +... +--- +name: recurrence_cant_reduce +tracksRegLiveness: true +body: | + ; CHECK-LABEL: name: recurrence_cant_reduce + ; CHECK: bb.0: + ; CHECK-NEXT: successors: %bb.1(0x80000000) + ; CHECK-NEXT: liveins: $x8, $x9 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %avl1:gprnox0 = COPY $x8 + ; CHECK-NEXT: %avl2:gprnox0 = COPY $x8 + ; CHECK-NEXT: %start:vr = PseudoVMV_V_I_M1 $noreg, 0, %avl1, 3 /* e8 */, 3 /* ta, ma */ + ; CHECK-NEXT: PseudoBR %bb.1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.1: + ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: %phi:vr = PHI %start, %bb.0, %inc, %bb.1 + ; CHECK-NEXT: %inc:vr = PseudoVADD_VI_M1 $noreg, %phi, 1, %avl1, 3 /* e8 */, 3 /* ta, ma */ + ; CHECK-NEXT: BNE $noreg, $noreg, %bb.1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: bb.2: + ; CHECK-NEXT: PseudoVSE8_V_M1 %inc, $noreg, %avl2, 3 /* e8 */ + bb.0: + liveins: $x8, $x9 + %avl1:gprnox0 = COPY $x8 + %avl2:gprnox0 = COPY $x8 + %start:vr = PseudoVMV_V_I_M1 $noreg, 0, -1, 3 /* e8 */, 3, /* ta, ma */ + PseudoBR %bb.1 + bb.1: + %phi:vr = PHI %start, %bb.0, %inc, %bb.1 + %inc:vr = PseudoVADD_VI_M1 $noreg, %phi, 1, %avl1, 3 /* e8 */, 3 /* ta, ma */ + BNE $noreg, $noreg, %bb.1 + bb.2: + PseudoVSE8_V_M1 %inc, $noreg, %avl2, 3 /* e8 */ +... From ea2b86183d8d0adc95e9d6314ee4a30ddd266dad Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Fri, 7 Feb 2025 11:37:56 +0800 Subject: [PATCH 02/12] [RISCV] Handle recurrences in RISCVVLOptimizer --- llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp | 132 +++++++++++------- .../CodeGen/RISCV/rvv/reproducer-pr146855.ll | 4 +- llvm/test/CodeGen/RISCV/rvv/vl-opt.ll | 3 +- llvm/test/CodeGen/RISCV/rvv/vl-opt.mir | 4 +- llvm/test/CodeGen/RISCV/rvv/vlopt-same-vl.ll | 4 +- 5 files changed, 92 insertions(+), 55 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp index f973e75840dc0..3ed4448915e02 100644 --- a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp +++ b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp @@ -30,6 +30,27 @@ using namespace llvm; namespace { +/// Wrapper around MachineOperand that defaults to immediate 0. +struct DemandedVL { + MachineOperand VL; + DemandedVL() : VL(MachineOperand::CreateImm(0)) {} + DemandedVL(MachineOperand VL) : VL(VL) {} + static DemandedVL vlmax() { + return DemandedVL(MachineOperand::CreateImm(RISCV::VLMaxSentinel)); + } + bool operator!=(const DemandedVL &Other) const { + return !VL.isIdenticalTo(Other.VL); + } +}; + +static DemandedVL max(const DemandedVL &LHS, const DemandedVL &RHS) { + if (RISCV::isVLKnownLE(LHS.VL, RHS.VL)) + return RHS; + if (RISCV::isVLKnownLE(RHS.VL, LHS.VL)) + return LHS; + return DemandedVL::vlmax(); +} + class RISCVVLOptimizer : public MachineFunctionPass { const MachineRegisterInfo *MRI; const MachineDominatorTree *MDT; @@ -51,17 +72,26 @@ class RISCVVLOptimizer : public MachineFunctionPass { StringRef getPassName() const override { return PASS_NAME; } private: - std::optional - getMinimumVLForUser(const MachineOperand &UserOp) const; - /// Returns the largest common VL MachineOperand that may be used to optimize - /// MI. Returns std::nullopt if it failed to find a suitable VL. - std::optional checkUsers(const MachineInstr &MI) const; + DemandedVL getMinimumVLForUser(const MachineOperand &UserOp) const; + /// Returns true if the users of \p MI have compatible EEWs and SEWs. + bool checkUsers(const MachineInstr &MI) const; bool tryReduceVL(MachineInstr &MI) const; bool isCandidate(const MachineInstr &MI) const; + void transfer(const MachineInstr &MI); + + /// Returns all uses of vector virtual registers. + auto vector_uses(const MachineInstr &MI) const { + auto Pred = [this](const MachineOperand &MO) -> bool { + return MO.isReg() && MO.getReg().isVirtual() && + RISCVRegisterInfo::isRVVRegClass(MRI->getRegClass(MO.getReg())); + }; + return make_filter_range(MI.uses(), Pred); + } /// For a given instruction, records what elements of it are demanded by /// downstream users. - DenseMap> DemandedVLs; + DenseMap DemandedVLs; + SetVector Worklist; }; /// Represents the EMUL and EEW of a MachineOperand. @@ -821,6 +851,9 @@ getOperandInfo(const MachineOperand &MO, const MachineRegisterInfo *MRI) { /// white-list approach simplifies this optimization for instructions that may /// have more complex semantics with relation to how it uses VL. static bool isSupportedInstr(const MachineInstr &MI) { + if (MI.isPHI() || MI.isFullCopy()) + return true; + const RISCVVPseudosTable::PseudoInfo *RVV = RISCVVPseudosTable::getPseudoInfo(MI.getOpcode()); @@ -1321,21 +1354,24 @@ bool RISCVVLOptimizer::isCandidate(const MachineInstr &MI) const { return true; } -std::optional +DemandedVL RISCVVLOptimizer::getMinimumVLForUser(const MachineOperand &UserOp) const { const MachineInstr &UserMI = *UserOp.getParent(); const MCInstrDesc &Desc = UserMI.getDesc(); + if (UserMI.isPHI() || UserMI.isFullCopy()) + return DemandedVLs.lookup(&UserMI); + if (!RISCVII::hasVLOp(Desc.TSFlags) || !RISCVII::hasSEWOp(Desc.TSFlags)) { LLVM_DEBUG(dbgs() << " Abort due to lack of VL, assume that" " use VLMAX\n"); - return std::nullopt; + return DemandedVL::vlmax(); } if (RISCVII::readsPastVL( TII->get(RISCV::getRVVMCOpcode(UserMI.getOpcode())).TSFlags)) { LLVM_DEBUG(dbgs() << " Abort because used by unsafe instruction\n"); - return std::nullopt; + return DemandedVL::vlmax(); } unsigned VLOpNum = RISCVII::getVLOpNum(Desc); @@ -1349,11 +1385,10 @@ RISCVVLOptimizer::getMinimumVLForUser(const MachineOperand &UserOp) const { if (UserOp.isTied()) { assert(UserOp.getOperandNo() == UserMI.getNumExplicitDefs() && RISCVII::isFirstDefTiedToFirstUse(UserMI.getDesc())); - auto DemandedVL = DemandedVLs.lookup(&UserMI); - if (!DemandedVL || !RISCV::isVLKnownLE(*DemandedVL, VLOp)) { + if (!RISCV::isVLKnownLE(DemandedVLs.lookup(&UserMI).VL, VLOp)) { LLVM_DEBUG(dbgs() << " Abort because user is passthru in " "instruction with demanded tail\n"); - return std::nullopt; + return DemandedVL::vlmax(); } } @@ -1366,18 +1401,16 @@ RISCVVLOptimizer::getMinimumVLForUser(const MachineOperand &UserOp) const { // If we know the demanded VL of UserMI, then we can reduce the VL it // requires. - if (auto DemandedVL = DemandedVLs.lookup(&UserMI)) { - assert(isCandidate(UserMI)); - if (RISCV::isVLKnownLE(*DemandedVL, VLOp)) - return DemandedVL; - } + if (RISCV::isVLKnownLE(DemandedVLs.lookup(&UserMI).VL, VLOp)) + return DemandedVLs.lookup(&UserMI); return VLOp; } -std::optional -RISCVVLOptimizer::checkUsers(const MachineInstr &MI) const { - std::optional CommonVL; +bool RISCVVLOptimizer::checkUsers(const MachineInstr &MI) const { + if (MI.isPHI() || MI.isFullCopy()) + return true; + SmallSetVector Worklist; SmallPtrSet PHISeen; for (auto &UserOp : MRI->use_operands(MI.getOperand(0).getReg())) @@ -1405,23 +1438,9 @@ RISCVVLOptimizer::checkUsers(const MachineInstr &MI) const { continue; } - auto VLOp = getMinimumVLForUser(UserOp); - if (!VLOp) - return std::nullopt; - - // Use the largest VL among all the users. If we cannot determine this - // statically, then we cannot optimize the VL. - if (!CommonVL || RISCV::isVLKnownLE(*CommonVL, *VLOp)) { - CommonVL = *VLOp; - LLVM_DEBUG(dbgs() << " User VL is: " << VLOp << "\n"); - } else if (!RISCV::isVLKnownLE(*VLOp, *CommonVL)) { - LLVM_DEBUG(dbgs() << " Abort because cannot determine a common VL\n"); - return std::nullopt; - } - if (!RISCVII::hasSEWOp(UserMI.getDesc().TSFlags)) { LLVM_DEBUG(dbgs() << " Abort due to lack of SEW operand\n"); - return std::nullopt; + return false; } std::optional ConsumerInfo = getOperandInfo(UserOp, MRI); @@ -1431,7 +1450,7 @@ RISCVVLOptimizer::checkUsers(const MachineInstr &MI) const { LLVM_DEBUG(dbgs() << " Abort due to unknown operand information.\n"); LLVM_DEBUG(dbgs() << " ConsumerInfo is: " << ConsumerInfo << "\n"); LLVM_DEBUG(dbgs() << " ProducerInfo is: " << ProducerInfo << "\n"); - return std::nullopt; + return false; } if (!OperandInfo::areCompatible(*ProducerInfo, *ConsumerInfo)) { @@ -1440,11 +1459,11 @@ RISCVVLOptimizer::checkUsers(const MachineInstr &MI) const { << " Abort due to incompatible information for EMUL or EEW.\n"); LLVM_DEBUG(dbgs() << " ConsumerInfo is: " << ConsumerInfo << "\n"); LLVM_DEBUG(dbgs() << " ProducerInfo is: " << ProducerInfo << "\n"); - return std::nullopt; + return false; } } - return CommonVL; + return true; } bool RISCVVLOptimizer::tryReduceVL(MachineInstr &MI) const { @@ -1460,9 +1479,7 @@ bool RISCVVLOptimizer::tryReduceVL(MachineInstr &MI) const { return false; } - auto CommonVL = DemandedVLs.lookup(&MI); - if (!CommonVL) - return false; + auto *CommonVL = &DemandedVLs.at(&MI).VL; assert((CommonVL->isImm() || CommonVL->getReg().isVirtual()) && "Expected VL to be an Imm or virtual Reg"); @@ -1497,6 +1514,24 @@ bool RISCVVLOptimizer::tryReduceVL(MachineInstr &MI) const { return true; } +static bool isPhysical(const MachineOperand &MO) { + return MO.isReg() && MO.getReg().isPhysical(); +} + +/// Look through \p MI's operands and propagate what it demands to its uses. +void RISCVVLOptimizer::transfer(const MachineInstr &MI) { + if (!isSupportedInstr(MI) || !checkUsers(MI) || any_of(MI.defs(), isPhysical)) + DemandedVLs[&MI] = DemandedVL::vlmax(); + + for (const MachineOperand &MO : vector_uses(MI)) { + const MachineInstr *Def = MRI->getVRegDef(MO.getReg()); + DemandedVL Prev = DemandedVLs[Def]; + DemandedVLs[Def] = max(DemandedVLs[Def], getMinimumVLForUser(MO)); + if (DemandedVLs[Def] != Prev) + Worklist.insert(Def); + } +} + bool RISCVVLOptimizer::runOnMachineFunction(MachineFunction &MF) { if (skipFunction(MF.getFunction())) return false; @@ -1513,14 +1548,17 @@ bool RISCVVLOptimizer::runOnMachineFunction(MachineFunction &MF) { assert(DemandedVLs.empty()); // For each instruction that defines a vector, compute what VL its - // downstream users demand. + // upstream uses demand. for (MachineBasicBlock *MBB : post_order(&MF)) { assert(MDT->isReachableFromEntry(MBB)); - for (MachineInstr &MI : reverse(*MBB)) { - if (!isCandidate(MI)) - continue; - DemandedVLs.insert({&MI, checkUsers(MI)}); - } + for (MachineInstr &MI : reverse(*MBB)) + Worklist.insert(&MI); + } + + while (!Worklist.empty()) { + const MachineInstr *MI = Worklist.front(); + Worklist.remove(MI); + transfer(*MI); } // Then go through and see if we can reduce the VL of any instructions to diff --git a/llvm/test/CodeGen/RISCV/rvv/reproducer-pr146855.ll b/llvm/test/CodeGen/RISCV/rvv/reproducer-pr146855.ll index cca00bf58063d..2d64defe8c7b1 100644 --- a/llvm/test/CodeGen/RISCV/rvv/reproducer-pr146855.ll +++ b/llvm/test/CodeGen/RISCV/rvv/reproducer-pr146855.ll @@ -6,7 +6,7 @@ target triple = "riscv64-unknown-linux-gnu" define i32 @_ZN4Mesh12rezone_countESt6vectorIiSaIiEERiS3_( %wide.load, %0, %1, %2, %3) #0 { ; CHECK-LABEL: _ZN4Mesh12rezone_countESt6vectorIiSaIiEERiS3_: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vsetvli a0, zero, e32, m2, ta, ma +; CHECK-NEXT: vsetivli zero, 0, e32, m2, ta, ma ; CHECK-NEXT: vmv1r.v v8, v0 ; CHECK-NEXT: li a0, 0 ; CHECK-NEXT: vmv.v.i v10, 0 @@ -14,7 +14,7 @@ define i32 @_ZN4Mesh12rezone_countESt6vectorIiSaIiEERiS3_( %wi ; CHECK-NEXT: vmv.v.i v14, 0 ; CHECK-NEXT: .LBB0_1: # %vector.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: vsetvli a1, zero, e32, m2, ta, mu +; CHECK-NEXT: vsetivli zero, 0, e32, m2, ta, mu ; CHECK-NEXT: vmv1r.v v0, v8 ; CHECK-NEXT: slli a0, a0, 2 ; CHECK-NEXT: vmv2r.v v16, v10 diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll b/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll index 8651eaad4ef28..ecea4efa4e768 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll @@ -202,7 +202,7 @@ define void @fadd_fcmp_select_copy( %v, %c define void @recurrence( %v, ptr %p, iXLen %n, iXLen %vl) { ; CHECK-LABEL: recurrence: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vsetvli a3, zero, e32, m2, ta, ma +; CHECK-NEXT: vsetvli zero, a2, e32, m2, ta, ma ; CHECK-NEXT: vmv.v.i v10, 0 ; CHECK-NEXT: .LBB13_1: # %loop ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 @@ -210,7 +210,6 @@ define void @recurrence( %v, ptr %p, iXLen %n, iXLen %vl) { ; CHECK-NEXT: vadd.vv v10, v10, v8 ; CHECK-NEXT: bnez a1, .LBB13_1 ; CHECK-NEXT: # %bb.2: # %exit -; CHECK-NEXT: vsetvli zero, a2, e32, m2, ta, ma ; CHECK-NEXT: vse32.v v10, (a0) ; CHECK-NEXT: ret entry: diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt.mir b/llvm/test/CodeGen/RISCV/rvv/vl-opt.mir index 538b68a70908b..2fcb8da339b06 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vl-opt.mir +++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt.mir @@ -613,14 +613,14 @@ body: | ; CHECK-NEXT: liveins: $x8 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: %avl:gprnox0 = COPY $x8 - ; CHECK-NEXT: %start:vr = PseudoVMV_V_I_M1 $noreg, 0, -1, 3 /* e8 */, 3 /* ta, ma */ + ; CHECK-NEXT: %start:vr = PseudoVMV_V_I_M1 $noreg, 0, %avl, 3 /* e8 */, 3 /* ta, ma */ ; CHECK-NEXT: PseudoBR %bb.1 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.1: ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000) ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: %phi:vr = PHI %start, %bb.0, %inc, %bb.1 - ; CHECK-NEXT: %inc:vr = PseudoVADD_VI_M1 $noreg, %phi, 1, -1, 3 /* e8 */, 3 /* ta, ma */ + ; CHECK-NEXT: %inc:vr = PseudoVADD_VI_M1 $noreg, %phi, 1, %avl, 3 /* e8 */, 3 /* ta, ma */ ; CHECK-NEXT: BNE $noreg, $noreg, %bb.1 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.2: diff --git a/llvm/test/CodeGen/RISCV/rvv/vlopt-same-vl.ll b/llvm/test/CodeGen/RISCV/rvv/vlopt-same-vl.ll index 4b9f9a0579c48..3a05477e64ccd 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vlopt-same-vl.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vlopt-same-vl.ll @@ -11,7 +11,7 @@ ; which was responsible for speeding it up. define @same_vl_imm( %passthru, %a, %b) { - ; CHECK: User VL is: 4 + ; CHECK: Trying to reduce VL for %{{.+}}:vrm2 = PseudoVADD_VV_M2 ; CHECK: Abort due to CommonVL == VLOp, no point in reducing. %v = call @llvm.riscv.vadd.nxv4i32.nxv4i32( poison, %a, %b, i64 4) %w = call @llvm.riscv.vadd.nxv4i32.nxv4i32( poison, %v, %a, i64 4) @@ -19,7 +19,7 @@ define @same_vl_imm( %passthru, @same_vl_reg( %passthru, %a, %b, i64 %vl) { - ; CHECK: User VL is: %3:gprnox0 + ; CHECK: Trying to reduce VL for %{{.+}}:vrm2 = PseudoVADD_VV_M2 ; CHECK: Abort due to CommonVL == VLOp, no point in reducing. %v = call @llvm.riscv.vadd.nxv4i32.nxv4i32( poison, %a, %b, i64 %vl) %w = call @llvm.riscv.vadd.nxv4i32.nxv4i32( poison, %v, %a, i64 %vl) From 9f24fe778aec374b3f2765fb27df17ae8c13fc56 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 30 Jul 2025 16:07:00 +0800 Subject: [PATCH 03/12] Link to talk in header comment --- llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp index 3ed4448915e02..5ee64719a923c 100644 --- a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp +++ b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp @@ -10,9 +10,19 @@ // instructions are inserted. // // The purpose of this optimization is to make the VL argument, for instructions -// that have a VL argument, as small as possible. This is implemented by -// visiting each instruction in reverse order and checking that if it has a VL -// argument, whether the VL can be reduced. +// that have a VL argument, as small as possible. +// +// This is split into a sparse dataflow analysis where we determine what VL is +// demanded by each instruction first, and then afterwards try to reduce the VL +// of each instruction if it demands less than its VL operand. +// +// The analysis is explained in more detail in the 2025 EuroLLVM Developers' +// Meeting talk "Accidental Dataflow Analysis: Extending the RISC-V VL +// Optimizer", which is available on YouTube at +// https://www.youtube.com/watch?v=Mfb5fRSdJAc +// +// The slides for the talk are available at +// https://llvm.org/devmtg/2025-04/slides/technical_talk/lau_accidental_dataflow.pdf // //===---------------------------------------------------------------------===// From 9b61df60f3922e55703c607c057fb940a21c806e Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Fri, 22 Aug 2025 09:42:41 +0800 Subject: [PATCH 04/12] Fix test name --- llvm/test/CodeGen/RISCV/rvv/vl-opt.ll | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll b/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll index ecea4efa4e768..0b2fba25f0a80 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll @@ -226,8 +226,8 @@ exit: ret void } -define @phi( %v, i1 %cond, iXLen %vl) { -; CHECK-LABEL: phi: +define @join( %v, i1 %cond, iXLen %vl) { +; CHECK-LABEL: join: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: andi a0, a0, 1 ; CHECK-NEXT: vsetivli zero, 2, e32, m2, ta, ma From 257ed3cfb2b68493c2935f7e92dd8e2c5ee19056 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 3 Sep 2025 15:11:04 +0800 Subject: [PATCH 05/12] Reword a comment to be more clear --- llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp index 75822c0ce3ac7..e431b15be78ca 100644 --- a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp +++ b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp @@ -1561,8 +1561,8 @@ bool RISCVVLOptimizer::runOnMachineFunction(MachineFunction &MF) { assert(DemandedVLs.empty()); - // For each instruction that defines a vector, compute what VL its - // upstream uses demand. + // For each instruction that defines a vector, propagate the VL it + // uses to its inputs. for (MachineBasicBlock *MBB : post_order(&MF)) { assert(MDT->isReachableFromEntry(MBB)); for (MachineInstr &MI : reverse(*MBB)) From 8f75db7e896b27395905e7e586feb06bbc978ad1 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Fri, 5 Sep 2025 14:59:52 +0800 Subject: [PATCH 06/12] Make vector_uses just a static function --- llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp index a302e3a7b28d8..49497ec77f7c5 100644 --- a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp +++ b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp @@ -89,15 +89,6 @@ class RISCVVLOptimizer : public MachineFunctionPass { bool isCandidate(const MachineInstr &MI) const; void transfer(const MachineInstr &MI); - /// Returns all uses of vector virtual registers. - auto vector_uses(const MachineInstr &MI) const { - auto Pred = [this](const MachineOperand &MO) -> bool { - return MO.isReg() && MO.getReg().isVirtual() && - RISCVRegisterInfo::isRVVRegClass(MRI->getRegClass(MO.getReg())); - }; - return make_filter_range(MI.uses(), Pred); - } - /// For a given instruction, records what elements of it are demanded by /// downstream users. DenseMap DemandedVLs; @@ -1634,12 +1625,18 @@ static bool isPhysical(const MachineOperand &MO) { return MO.isReg() && MO.getReg().isPhysical(); } +static bool isVirtualVec(const MachineOperand &MO) { + return MO.isReg() && MO.getReg().isVirtual() && + RISCVRegisterInfo::isRVVRegClass( + MO.getParent()->getMF()->getRegInfo().getRegClass(MO.getReg())); +} + /// Look through \p MI's operands and propagate what it demands to its uses. void RISCVVLOptimizer::transfer(const MachineInstr &MI) { if (!isSupportedInstr(MI) || !checkUsers(MI) || any_of(MI.defs(), isPhysical)) DemandedVLs[&MI] = DemandedVL::vlmax(); - for (const MachineOperand &MO : vector_uses(MI)) { + for (const MachineOperand &MO : make_filter_range(MI.uses(), isVirtualVec)) { const MachineInstr *Def = MRI->getVRegDef(MO.getReg()); DemandedVL Prev = DemandedVLs[Def]; DemandedVLs[Def] = max(DemandedVLs[Def], getMinimumVLForUser(MO)); From d550870b5685cce86d8091eed84737b03e76b9ea Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Fri, 5 Sep 2025 16:22:05 +0800 Subject: [PATCH 07/12] clang-format --- llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp index 49497ec77f7c5..6c5954f1b7056 100644 --- a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp +++ b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp @@ -843,7 +843,7 @@ static std::optional getOperandLog2EEW(const MachineOperand &MO) { static std::optional getOperandInfo(const MachineOperand &MO) { const MachineInstr &MI = *MO.getParent(); const RISCVVPseudosTable::PseudoInfo *RVV = - RISCVVPseudosTable::getPseudoInfo(MI.getOpcode()); + RISCVVPseudosTable::getPseudoInfo(MI.getOpcode()); MI.dump(); assert(RVV && "Could not find MI in PseudoTable"); @@ -1486,8 +1486,7 @@ static bool isSegmentedStoreInstr(const MachineInstr &MI) { } } -bool -RISCVVLOptimizer::checkUsers(const MachineInstr &MI) const { +bool RISCVVLOptimizer::checkUsers(const MachineInstr &MI) const { if (MI.isPHI() || MI.isFullCopy() || isTupleInsertInstr(MI)) return true; From dc0ca0eadbd1528d850263768ab8e263a4b52e3e Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Sat, 6 Sep 2025 11:44:43 +0800 Subject: [PATCH 08/12] Remove debug code --- llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp index 6c5954f1b7056..ad4828525f1db 100644 --- a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp +++ b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp @@ -844,7 +844,6 @@ static std::optional getOperandInfo(const MachineOperand &MO) { const MachineInstr &MI = *MO.getParent(); const RISCVVPseudosTable::PseudoInfo *RVV = RISCVVPseudosTable::getPseudoInfo(MI.getOpcode()); - MI.dump(); assert(RVV && "Could not find MI in PseudoTable"); std::optional Log2EEW = getOperandLog2EEW(MO); From ebce54610b162b3038663f7f21320053d472e37b Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Thu, 11 Sep 2025 10:12:34 +0800 Subject: [PATCH 09/12] Add vleff recurrence test case --- llvm/test/CodeGen/RISCV/rvv/vl-opt.ll | 39 +++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll b/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll index eab9285b19a4f..3844b984455c4 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt.ll @@ -266,18 +266,53 @@ exit: ret void } +define void @recurrence_vleff( %v, ptr %p, iXLen %n, iXLen %vl) { +; CHECK-LABEL: recurrence_vleff: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vsetvli zero, a2, e32, m2, ta, ma +; CHECK-NEXT: vmv.v.i v8, 0 +; CHECK-NEXT: mv a3, a0 +; CHECK-NEXT: .LBB17_1: # %loop +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: vsetvli zero, a2, e32, m2, ta, ma +; CHECK-NEXT: vle32ff.v v10, (a3) +; CHECK-NEXT: addi a1, a1, -1 +; CHECK-NEXT: vadd.vv v8, v8, v10 +; CHECK-NEXT: vse32.v v8, (a0) +; CHECK-NEXT: addi a3, a3, 4 +; CHECK-NEXT: bnez a1, .LBB17_1 +; CHECK-NEXT: # %bb.2: # %exit +; CHECK-NEXT: ret +entry: + br label %loop +loop: + %iv = phi iXLen [ 0, %entry ], [ %iv.next, %loop ] + %phi = phi [ zeroinitializer, %entry ], [ %y, %loop ] + %gep = getelementptr i32, ptr %p, iXLen %iv + %vleff = call { , iXLen } @llvm.riscv.vleff( poison, ptr %gep, iXLen %vl) + %vleff.x = extractvalue { , iXLen } %vleff, 0 + %vleff.vl = extractvalue { , iXLen } %vleff, 1 + %y = add %phi, %vleff.x + call void @llvm.riscv.vse( %y, ptr %p, iXLen %vleff.vl) + %iv.next = add iXLen %iv, 1 + %done = icmp eq iXLen %iv.next, %n + br i1 %done, label %exit, label %loop +exit: + ret void +} + define @join( %v, i1 %cond, iXLen %vl) { ; CHECK-LABEL: join: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: andi a0, a0, 1 ; CHECK-NEXT: vsetivli zero, 2, e32, m2, ta, ma ; CHECK-NEXT: vadd.vi v8, v8, 1 -; CHECK-NEXT: beqz a0, .LBB17_2 +; CHECK-NEXT: beqz a0, .LBB18_2 ; CHECK-NEXT: # %bb.1: # %foo ; CHECK-NEXT: vsetivli zero, 1, e32, m2, ta, ma ; CHECK-NEXT: vadd.vi v8, v8, 1 ; CHECK-NEXT: ret -; CHECK-NEXT: .LBB17_2: # %bar +; CHECK-NEXT: .LBB18_2: # %bar ; CHECK-NEXT: vadd.vi v8, v8, 2 ; CHECK-NEXT: ret entry: From cadf39301ba9b699c43ab461e1eaf65856101491 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Fri, 12 Sep 2025 14:02:15 +0800 Subject: [PATCH 10/12] Move max into DemandedVL --- llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp index ad4828525f1db..be4699eaa79f8 100644 --- a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp +++ b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp @@ -51,15 +51,15 @@ struct DemandedVL { bool operator!=(const DemandedVL &Other) const { return !VL.isIdenticalTo(Other.VL); } -}; -static DemandedVL max(const DemandedVL &LHS, const DemandedVL &RHS) { - if (RISCV::isVLKnownLE(LHS.VL, RHS.VL)) - return RHS; - if (RISCV::isVLKnownLE(RHS.VL, LHS.VL)) - return LHS; - return DemandedVL::vlmax(); -} + DemandedVL max(const DemandedVL &X) const { + if (RISCV::isVLKnownLE(VL, X.VL)) + return X; + if (RISCV::isVLKnownLE(X.VL, VL)) + return *this; + return DemandedVL::vlmax(); + } +}; class RISCVVLOptimizer : public MachineFunctionPass { const MachineRegisterInfo *MRI; @@ -1637,7 +1637,7 @@ void RISCVVLOptimizer::transfer(const MachineInstr &MI) { for (const MachineOperand &MO : make_filter_range(MI.uses(), isVirtualVec)) { const MachineInstr *Def = MRI->getVRegDef(MO.getReg()); DemandedVL Prev = DemandedVLs[Def]; - DemandedVLs[Def] = max(DemandedVLs[Def], getMinimumVLForUser(MO)); + DemandedVLs[Def] = DemandedVLs[Def].max(getMinimumVLForUser(MO)); if (DemandedVLs[Def] != Prev) Worklist.insert(Def); } From 0ef2baf37a9772b80c4ae24ae8b67b1d9ee8491c Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Fri, 12 Sep 2025 14:13:57 +0800 Subject: [PATCH 11/12] Move isVirtualVec into virtual_vec_uses method, avoid pointer chasing --- llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp index be4699eaa79f8..e655d08234b08 100644 --- a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp +++ b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp @@ -93,6 +93,14 @@ class RISCVVLOptimizer : public MachineFunctionPass { /// downstream users. DenseMap DemandedVLs; SetVector Worklist; + + /// \returns all vector virtual registers that \p MI uses. + auto virtual_vec_uses(const MachineInstr &MI) const { + return make_filter_range(MI.uses(), [this](const MachineOperand &MO) { + return MO.isReg() && MO.getReg().isVirtual() && + RISCVRegisterInfo::isRVVRegClass(MRI->getRegClass(MO.getReg())); + }); + } }; /// Represents the EMUL and EEW of a MachineOperand. @@ -1623,18 +1631,12 @@ static bool isPhysical(const MachineOperand &MO) { return MO.isReg() && MO.getReg().isPhysical(); } -static bool isVirtualVec(const MachineOperand &MO) { - return MO.isReg() && MO.getReg().isVirtual() && - RISCVRegisterInfo::isRVVRegClass( - MO.getParent()->getMF()->getRegInfo().getRegClass(MO.getReg())); -} - /// Look through \p MI's operands and propagate what it demands to its uses. void RISCVVLOptimizer::transfer(const MachineInstr &MI) { if (!isSupportedInstr(MI) || !checkUsers(MI) || any_of(MI.defs(), isPhysical)) DemandedVLs[&MI] = DemandedVL::vlmax(); - for (const MachineOperand &MO : make_filter_range(MI.uses(), isVirtualVec)) { + for (const MachineOperand &MO : virtual_vec_uses(MI)) { const MachineInstr *Def = MRI->getVRegDef(MO.getReg()); DemandedVL Prev = DemandedVLs[Def]; DemandedVLs[Def] = DemandedVLs[Def].max(getMinimumVLForUser(MO)); From 1068f3aa9d666d86186ca793b4d926d7b99da91f Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Fri, 12 Sep 2025 14:17:43 +0800 Subject: [PATCH 12/12] Rename Worklist in checkUsers to avoid shadowing --- llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp index e655d08234b08..a1134663c0e7a 100644 --- a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp +++ b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp @@ -1497,19 +1497,19 @@ bool RISCVVLOptimizer::checkUsers(const MachineInstr &MI) const { if (MI.isPHI() || MI.isFullCopy() || isTupleInsertInstr(MI)) return true; - SmallSetVector Worklist; + SmallSetVector OpWorklist; SmallPtrSet PHISeen; for (auto &UserOp : MRI->use_operands(MI.getOperand(0).getReg())) - Worklist.insert(&UserOp); + OpWorklist.insert(&UserOp); - while (!Worklist.empty()) { - MachineOperand &UserOp = *Worklist.pop_back_val(); + while (!OpWorklist.empty()) { + MachineOperand &UserOp = *OpWorklist.pop_back_val(); const MachineInstr &UserMI = *UserOp.getParent(); LLVM_DEBUG(dbgs() << " Checking user: " << UserMI << "\n"); if (UserMI.isFullCopy() && UserMI.getOperand(0).getReg().isVirtual()) { LLVM_DEBUG(dbgs() << " Peeking through uses of COPY\n"); - Worklist.insert_range(llvm::make_pointer_range( + OpWorklist.insert_range(llvm::make_pointer_range( MRI->use_operands(UserMI.getOperand(0).getReg()))); continue; } @@ -1526,7 +1526,7 @@ bool RISCVVLOptimizer::checkUsers(const MachineInstr &MI) const { if (!isTupleInsertInstr(CandidateMI) && !isSegmentedStoreInstr(CandidateMI)) return false; - Worklist.insert(&UseOp); + OpWorklist.insert(&UseOp); } continue; } @@ -1536,7 +1536,7 @@ bool RISCVVLOptimizer::checkUsers(const MachineInstr &MI) const { if (!PHISeen.insert(&UserMI).second) continue; LLVM_DEBUG(dbgs() << " Peeking through uses of PHI\n"); - Worklist.insert_range(llvm::make_pointer_range( + OpWorklist.insert_range(llvm::make_pointer_range( MRI->use_operands(UserMI.getOperand(0).getReg()))); continue; }