From 896ef734f373e76ee54b07aa9aa35d53b83ce87b Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Thu, 19 Feb 2026 20:16:15 +0800 Subject: [PATCH 1/9] [VPlan] Assert vplan-verify-each result and get verifier passing Currently if -vplan-verify-each is enabled and a pass fails the verifier, it will output the failure to stderr but will still finish with a zero exit code. This adds an assert that the verification fails so that e.g. lit will pick up verifier failures in the in-tree tests with an EXPENSIVE_CHECKS build. However currently the verifier fails in several tests, so this also includes several fixes: 1. Remove the EVL verifier checks. The EVL is used in quite a few more places than when the verification was originally added, and nowadays it gets used in a variety of VPInstructions, all of which need to be handled by the verifier. There are a few passes that fail today because we haven't updated the EVL verification. Now that the EVL transform has been split up into a variable stepping transform and an optimization pass it's generally less fragile, so there is less value in this verification. 2. Extend the LastActiveLane verification to handle more prefix masks. All of the prefix masks that the verifier encounters are of the form `icmp ult/ule monotonically-increasing-sequence, uniform`, which always generate a prefix mask. 3. Allow multiple VPActiveLaneMaskPHIRecipe recipes after unrolling. This also allows us to remove the verifyLate argument Tested that llvm-test-suite + SPEC CPU 2017 now pass with -vplan-verify-each enabled for RISC-V. --- .../Transforms/Vectorize/LoopVectorize.cpp | 5 +- .../Transforms/Vectorize/VPlanTransforms.h | 6 +- .../Transforms/Vectorize/VPlanVerifier.cpp | 151 +++--------------- llvm/lib/Transforms/Vectorize/VPlanVerifier.h | 6 +- 4 files changed, 34 insertions(+), 134 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 01433fe4c4ba7..68bdb9f4ba23f 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -364,7 +364,7 @@ cl::opt cl::init(false), #endif cl::Hidden, - cl::desc("Verfiy VPlans after VPlan transforms.")); + cl::desc("Verify VPlans after VPlan transforms.")); #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) cl::opt llvm::VPlanPrintAfterAll( @@ -7512,8 +7512,7 @@ DenseMap LoopVectorizationPlanner::executePlan( State.CFG.PrevBB->getSingleSuccessor(), &BestVPlan); VPlanTransforms::removeDeadRecipes(BestVPlan); - assert(verifyVPlanIsValid(BestVPlan, true /*VerifyLate*/) && - "final VPlan is invalid"); + assert(verifyVPlanIsValid(BestVPlan) && "final VPlan is invalid"); // After vectorization, the exit blocks of the original loop will have // additional predecessors. Invalidate SCEVs for the exit phis in case SE diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index f2dfc166cecc9..e35872bbdff37 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -72,8 +72,10 @@ struct VPlanTransforms { dbgs() << Plan << '\n'; } #endif - if (VerifyEachVPlan && EnableVerify) - verifyVPlanIsValid(Plan); + if (VerifyEachVPlan && EnableVerify) { + [[maybe_unused]] bool IsValid = verifyVPlanIsValid(Plan); + assert(IsValid && "VPlan is invalid"); + } }}; return std::forward(Pass)(Plan, std::forward(Args)...); diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index e4b334c3eba49..2db7287d494f6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -31,7 +31,6 @@ namespace { class VPlanVerifier { const VPDominatorTree &VPDT; VPTypeAnalysis &TypeInfo; - bool VerifyLate; SmallPtrSet WrappedIRBBs; @@ -40,11 +39,6 @@ class VPlanVerifier { // VPHeaderPHIRecipes. bool verifyPhiRecipes(const VPBasicBlock *VPBB); - /// Verify that \p EVL is used correctly. The user must be either in - /// EVL-based recipes as a last operand or VPInstruction::Add which is - /// incoming value into EVL's recipe. - bool verifyEVLRecipe(const VPInstruction &EVL) const; - /// Verify that \p LastActiveLane's operand is guaranteed to be a prefix-mask. bool verifyLastActiveLaneRecipe(const VPInstruction &LastActiveLane) const; @@ -67,9 +61,8 @@ class VPlanVerifier { bool verifyRegionRec(const VPRegionBlock *Region); public: - VPlanVerifier(VPDominatorTree &VPDT, VPTypeAnalysis &TypeInfo, - bool VerifyLate) - : VPDT(VPDT), TypeInfo(TypeInfo), VerifyLate(VerifyLate) {} + VPlanVerifier(VPDominatorTree &VPDT, VPTypeAnalysis &TypeInfo) + : VPDT(VPDT), TypeInfo(TypeInfo) {} bool verify(const VPlan &Plan); }; @@ -124,7 +117,7 @@ bool VPlanVerifier::verifyPhiRecipes(const VPBasicBlock *VPBB) { RecipeI++; } - if (!VerifyLate && NumActiveLaneMaskPhiRecipes > 1) { + if (!VPBB->getPlan()->isUnrolled() && NumActiveLaneMaskPhiRecipes > 1) { errs() << "There should be no more than one VPActiveLaneMaskPHIRecipe"; return false; } @@ -146,106 +139,19 @@ bool VPlanVerifier::verifyPhiRecipes(const VPBasicBlock *VPBB) { return true; } -bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const { - if (EVL.getOpcode() != VPInstruction::ExplicitVectorLength) { - errs() << "verifyEVLRecipe should only be called on " - "VPInstruction::ExplicitVectorLength\n"; - return false; - } - auto VerifyEVLUse = [&](const VPRecipeBase &R, - const unsigned ExpectedIdx) -> bool { - SmallVector Ops(R.operands()); - unsigned UseCount = count(Ops, &EVL); - if (UseCount != 1 || Ops[ExpectedIdx] != &EVL) { - errs() << "EVL is used as non-last operand in EVL-based recipe\n"; - return false; - } +static bool isKnownMonotonic(VPValue *V) { + VPValue *X, *Y; + if (match(V, m_Add(m_VPValue(X), m_VPValue(Y)))) + return isKnownMonotonic(X) && isKnownMonotonic(Y); + if (match(V, m_StepVector())) return true; - }; - auto VerifyEVLUseInVecEndPtr = [&EVL](auto &VEPRs) { - if (all_of(VEPRs, [&EVL](VPUser *U) { - auto *VEPR = cast(U); - return match(VEPR->getOffset(), - m_c_Mul(m_SpecificSInt(VEPR->getStride()), - m_Sub(m_Specific(&EVL), m_One()))); - })) - return true; - errs() << "Expected VectorEndPointer with EVL operand\n"; - return false; - }; - return all_of(EVL.users(), [&](VPUser *U) { - return TypeSwitch(U) - .Case([&](const VPWidenIntrinsicRecipe *S) { - return VerifyEVLUse(*S, S->getNumOperands() - 1); - }) - .Case( - [&](const VPRecipeBase *S) { return VerifyEVLUse(*S, 2); }) - .Case([&](const VPScalarIVStepsRecipe *R) { - if (R->getNumOperands() != 3) { - errs() << "Unrolling with EVL tail folding not yet supported\n"; - return false; - } - return VerifyEVLUse(*R, 2); - }) - .Case( - [&](const VPRecipeBase *R) { return VerifyEVLUse(*R, 1); }) - .Case( - [&](const VPInstructionWithType *S) { return VerifyEVLUse(*S, 0); }) - .Case([&](const VPInstruction *I) { - if (I->getOpcode() == Instruction::PHI || - I->getOpcode() == Instruction::ICmp) - return VerifyEVLUse(*I, 1); - if (I->getOpcode() == Instruction::Sub) { - // If Sub has a single user that's a SingleDefRecipe (which is - // expected to be a Mul), filter its users, in turn, to get - // VectorEndPointerRecipes, and verify that all the offsets match - // (EVL - 1) * Stride. - if (auto *Def = dyn_cast_if_present( - I->getSingleUser())) { - auto VEPRs = make_filter_range(Def->users(), - IsaPred); - if (!VEPRs.empty()) - return VerifyEVLUseInVecEndPtr(VEPRs); - } - return VerifyEVLUse(*I, 1); - } - switch (I->getOpcode()) { - case Instruction::Add: - break; - case Instruction::UIToFP: - case Instruction::Trunc: - case Instruction::ZExt: - case Instruction::Mul: - case Instruction::Shl: - case Instruction::FMul: - case VPInstruction::Broadcast: - case VPInstruction::PtrAdd: - // Opcodes above can only use EVL after wide inductions have been - // expanded. - if (!VerifyLate) { - errs() << "EVL used by unexpected VPInstruction\n"; - return false; - } - break; - default: - errs() << "EVL used by unexpected VPInstruction\n"; - return false; - } - if (!VerifyLate && - !isa(*I->users().begin())) { - errs() << "Result of VPInstruction::Add with EVL operand is " - "not used by VPCurrentIterationPHIRecipe\n"; - return false; - } - return true; - }) - .Default([&](const VPUser *U) { - errs() << "EVL has unexpected user\n"; - return false; - }); - }); + if (auto *WidenIV = dyn_cast(V)) + return match(WidenIV->getStepValue(), m_One()); + if (auto *WidenIV = dyn_cast(V)) + return match(WidenIV->getStepValue(), m_One()); + if (isa(V)) + return true; + return vputils::isUniformAcrossVFsAndUFs(V); } bool VPlanVerifier::verifyLastActiveLaneRecipe( @@ -259,18 +165,19 @@ bool VPlanVerifier::verifyLastActiveLaneRecipe( } const VPlan &Plan = *LastActiveLane.getParent()->getPlan(); - // All operands must be prefix-mask. Currently we check for header masks or - // EVL-derived masks, as those are currently the only operands in practice, - // but this may need updating in the future. + // All operands must be prefix-mask. This means an icmp ult/ule LHS, RHS where + // the LHS is monotonically increasing and RHS is uniform. for (VPValue *Op : LastActiveLane.operands()) { if (vputils::isHeaderMask(Op, Plan)) continue; - // Masks derived from EVL are also fine. - auto BroadcastOrEVL = - m_CombineOr(m_Broadcast(m_EVL(m_VPValue())), m_EVL(m_VPValue())); - if (match(Op, m_CombineOr(m_ICmp(m_StepVector(), BroadcastOrEVL), - m_ICmp(BroadcastOrEVL, m_StepVector())))) + CmpPredicate Pred; + VPValue *LHS, *RHS; + if (match(Op, m_ICmp(Pred, m_VPValue(LHS), m_VPValue(RHS))) && + (Pred == CmpInst::ICMP_ULE || Pred == CmpInst::ICMP_ULT) && + isKnownMonotonic(LHS) && + (vputils::isUniformAcrossVFsAndUFs(RHS) || + vputils::isSingleScalar(RHS))) continue; errs() << "LastActiveLane operand "; @@ -372,12 +279,6 @@ bool VPlanVerifier::verifyVPBasicBlock(const VPBasicBlock *VPBB) { } if (const auto *VPI = dyn_cast(&R)) { switch (VPI->getOpcode()) { - case VPInstruction::ExplicitVectorLength: - if (!verifyEVLRecipe(*VPI)) { - errs() << "EVL VPValue is not used correctly\n"; - return false; - } - break; case VPInstruction::LastActiveLane: if (!verifyLastActiveLaneRecipe(*VPI)) return false; @@ -569,9 +470,9 @@ bool VPlanVerifier::verify(const VPlan &Plan) { return true; } -bool llvm::verifyVPlanIsValid(const VPlan &Plan, bool VerifyLate) { +bool llvm::verifyVPlanIsValid(const VPlan &Plan) { VPDominatorTree VPDT(const_cast(Plan)); VPTypeAnalysis TypeInfo(Plan); - VPlanVerifier Verifier(VPDT, TypeInfo, VerifyLate); + VPlanVerifier Verifier(VPDT, TypeInfo); return Verifier.verify(Plan); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.h b/llvm/lib/Transforms/Vectorize/VPlanVerifier.h index ccf79e8e5c985..642b25b0635c0 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.h +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.h @@ -29,16 +29,14 @@ namespace llvm { class VPlan; -/// Verify invariants for general VPlans. If \p VerifyLate is passed, skip some -/// checks that are not applicable at later stages of the transform pipeline. +/// Verify invariants for general VPlans. /// Currently it checks the following: /// 1. Region/Block verification: Check the Region/Block verification /// invariants for every region in the H-CFG. /// 2. all phi-like recipes must be at the beginning of a block, with no other /// recipes in between. Note that currently there is still an exception for /// VPBlendRecipes. -LLVM_ABI_FOR_TEST bool verifyVPlanIsValid(const VPlan &Plan, - bool VerifyLate = false); +LLVM_ABI_FOR_TEST bool verifyVPlanIsValid(const VPlan &Plan); } // namespace llvm From 9401b48051b8e34694bb3991e159dc241bcbfeba Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Fri, 20 Feb 2026 00:09:30 +0800 Subject: [PATCH 2/9] Use report_fatal_error --- llvm/lib/Transforms/Vectorize/VPlanTransforms.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index e35872bbdff37..c24aa128ba61b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -73,8 +73,8 @@ struct VPlanTransforms { } #endif if (VerifyEachVPlan && EnableVerify) { - [[maybe_unused]] bool IsValid = verifyVPlanIsValid(Plan); - assert(IsValid && "VPlan is invalid"); + if (!verifyVPlanIsValid(Plan)) + report_fatal_error("Broken VPlan found, compilation aborted!"); } }}; From e9059efeb6262adbdbf8052a41ed5487ad79ff0b Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Fri, 20 Feb 2026 00:47:24 +0800 Subject: [PATCH 3/9] Tighten monotonic IV checks to cases where there's no known overflow --- llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h | 2 ++ llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp | 11 ++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h index db229300af17a..7ba302733762a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h +++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h @@ -859,6 +859,8 @@ inline auto m_c_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) { return m_c_Select(Op0, m_True(), Op1); } +inline auto m_CanonicalIV() { return class_match(); } + template using VPScalarIVSteps_match = Recipe_match, 0, false, VPScalarIVStepsRecipe>; diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index 2db7287d494f6..e2a9875515646 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -146,9 +146,14 @@ static bool isKnownMonotonic(VPValue *V) { if (match(V, m_StepVector())) return true; if (auto *WidenIV = dyn_cast(V)) - return match(WidenIV->getStepValue(), m_One()); - if (auto *WidenIV = dyn_cast(V)) - return match(WidenIV->getStepValue(), m_One()); + return match(WidenIV->getStartValue(), m_ZeroInt()) && + match(WidenIV->getStepValue(), m_One()); + if (auto *Steps = dyn_cast(V)) + return match(Steps->getOperand(0), + m_CombineOr( + m_CanonicalIV(), + m_DerivedIV(m_ZeroInt(), m_CanonicalIV(), m_One()))) && + match(Steps->getStepValue(), m_One()); if (isa(V)) return true; return vputils::isUniformAcrossVFsAndUFs(V); From ba01b6326ebd0fbad011dd1b298a3c3b9ed93955 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Fri, 20 Feb 2026 01:23:34 +0800 Subject: [PATCH 4/9] Add comment --- llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index e2a9875515646..a23e0a7e50635 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -145,6 +145,7 @@ static bool isKnownMonotonic(VPValue *V) { return isKnownMonotonic(X) && isKnownMonotonic(Y); if (match(V, m_StepVector())) return true; + // Only handle a subset of IVs until we can guarantee there's no overflow. if (auto *WidenIV = dyn_cast(V)) return match(WidenIV->getStartValue(), m_ZeroInt()) && match(WidenIV->getStepValue(), m_One()); From db8dbb3c5462623e55699507a30c2bfea98f911b Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Fri, 20 Feb 2026 12:12:45 +0800 Subject: [PATCH 5/9] Check for nuw on add --- llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index a23e0a7e50635..005ca1e0e3943 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -141,7 +141,8 @@ bool VPlanVerifier::verifyPhiRecipes(const VPBasicBlock *VPBB) { static bool isKnownMonotonic(VPValue *V) { VPValue *X, *Y; - if (match(V, m_Add(m_VPValue(X), m_VPValue(Y)))) + if (match(V, m_Add(m_VPValue(X), m_VPValue(Y))) && + cast(V)->hasNoUnsignedWrap()) return isKnownMonotonic(X) && isKnownMonotonic(Y); if (match(V, m_StepVector())) return true; From 08155c7c6dd7b8981f998d277f116fcf5670e579 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Tue, 24 Feb 2026 20:17:08 +0800 Subject: [PATCH 6/9] Fix accidentally dropping recursive m_Add monotonic check, use isCanonical --- llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index 005ca1e0e3943..28996fa062fea 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -141,15 +141,14 @@ bool VPlanVerifier::verifyPhiRecipes(const VPBasicBlock *VPBB) { static bool isKnownMonotonic(VPValue *V) { VPValue *X, *Y; - if (match(V, m_Add(m_VPValue(X), m_VPValue(Y))) && - cast(V)->hasNoUnsignedWrap()) - return isKnownMonotonic(X) && isKnownMonotonic(Y); + if (match(V, m_Add(m_VPValue(X), m_VPValue(Y)))) + cast(V)->hasNoUnsignedWrap() && isKnownMonotonic(X) && + isKnownMonotonic(Y); if (match(V, m_StepVector())) return true; // Only handle a subset of IVs until we can guarantee there's no overflow. if (auto *WidenIV = dyn_cast(V)) - return match(WidenIV->getStartValue(), m_ZeroInt()) && - match(WidenIV->getStepValue(), m_One()); + return WidenIV->isCanonical(); if (auto *Steps = dyn_cast(V)) return match(Steps->getOperand(0), m_CombineOr( From b50323159cd2ee655e691e9095f841f17d6d0edc Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Tue, 24 Feb 2026 21:58:36 +0800 Subject: [PATCH 7/9] Remove noUnsignedWrap check, check for EVL explicitly --- llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index 28996fa062fea..3a1c3895c3fca 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -141,9 +141,9 @@ bool VPlanVerifier::verifyPhiRecipes(const VPBasicBlock *VPBB) { static bool isKnownMonotonic(VPValue *V) { VPValue *X, *Y; + // TODO: Check for hasNoUnsignedWrap() when we set nuw in VPlanUnroll if (match(V, m_Add(m_VPValue(X), m_VPValue(Y)))) - cast(V)->hasNoUnsignedWrap() && isKnownMonotonic(X) && - isKnownMonotonic(Y); + return isKnownMonotonic(X) && isKnownMonotonic(Y); if (match(V, m_StepVector())) return true; // Only handle a subset of IVs until we can guarantee there's no overflow. @@ -183,7 +183,7 @@ bool VPlanVerifier::verifyLastActiveLaneRecipe( (Pred == CmpInst::ICMP_ULE || Pred == CmpInst::ICMP_ULT) && isKnownMonotonic(LHS) && (vputils::isUniformAcrossVFsAndUFs(RHS) || - vputils::isSingleScalar(RHS))) + match(RHS, m_EVL(m_VPValue())))) continue; errs() << "LastActiveLane operand "; From 0f6242802b851a48b31103870ce28eff28c3ce97 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 25 Feb 2026 22:48:04 +0800 Subject: [PATCH 8/9] Update comment, add unit test --- .../Transforms/Vectorize/VPlanVerifier.cpp | 2 +- .../Vectorize/VPlanVerifierTest.cpp | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index 3a1c3895c3fca..d62a71883b6ea 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -172,7 +172,7 @@ bool VPlanVerifier::verifyLastActiveLaneRecipe( const VPlan &Plan = *LastActiveLane.getParent()->getPlan(); // All operands must be prefix-mask. This means an icmp ult/ule LHS, RHS where - // the LHS is monotonically increasing and RHS is uniform. + // the LHS is monotonically increasing and RHS is uniform across VFs and UF. for (VPValue *Op : LastActiveLane.operands()) { if (vputils::isHeaderMask(Op, Plan)) continue; diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp index f07ee94105ff8..c6f4e9aec31bf 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp @@ -16,6 +16,10 @@ using namespace llvm; +namespace llvm { +LLVM_ABI extern cl::opt VerifyEachVPlan; +} // namespace llvm + using VPVerifierTest = VPlanTestBase; namespace { @@ -342,6 +346,28 @@ TEST_F(VPVerifierTest, NonHeaderPHIInHeader) { delete PHINode; } +TEST_F(VPVerifierTest, testRUN_VPLAN_PASS) { + VPlan &Plan = getPlan(); + VPIRValue *Zero = Plan.getConstantInt(32, 0); + VPInstruction *DefI = + new VPInstruction(Instruction::Add, {Zero, Zero}, + VPIRFlags::getDefaultFlags(Instruction::Add)); + VPInstruction *UseI = + new VPInstruction(Instruction::Sub, {DefI, Zero}, + VPIRFlags::getDefaultFlags(Instruction::Sub)); + VPBasicBlock *VPBB1 = Plan.getEntry(); + VPBB1->appendRecipe(UseI); + VPBB1->appendRecipe(DefI); + + bool OrigVerifyEachVPlan = VerifyEachVPlan; + VerifyEachVPlan = true; + llvm::scope_exit _([&]() { VerifyEachVPlan = OrigVerifyEachVPlan; }); + auto NopPass = [](VPlan &Plan) {}; + EXPECT_DEATH( + { VPlanTransforms::runPass("simplifyRecipes", NopPass, Plan); }, + "Broken VPlan found, compilation aborted!"); +} + class VPIRVerifierTest : public VPlanTestIRBase {}; TEST_F(VPIRVerifierTest, testVerifyIRPhiInScalarHeaderVPIRBB) { @@ -418,4 +444,5 @@ TEST_F(VPIRVerifierTest, testVerifyIRPhiInExitVPIRBB) { ::testing::internal::GetCapturedStderr().c_str()); #endif } + } // namespace From 84eb19718113755fcc82a28092b825e380fddeed Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 25 Feb 2026 22:55:18 +0800 Subject: [PATCH 9/9] Undo stray whitespace change --- llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp index c6f4e9aec31bf..2472e18b654e2 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp @@ -444,5 +444,4 @@ TEST_F(VPIRVerifierTest, testVerifyIRPhiInExitVPIRBB) { ::testing::internal::GetCapturedStderr().c_str()); #endif } - } // namespace