diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 0fd425c23c7aa..b0cbef6227388 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -364,7 +364,7 @@ cl::opt cl::init(false), #endif cl::Hidden, - cl::desc("Verfiy VPlans after VPlan transforms.")); + cl::desc("Verify VPlans after VPlan transforms.")); #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) cl::opt llvm::VPlanPrintAfterAll( diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h index db229300af17a..7ba302733762a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h +++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h @@ -859,6 +859,8 @@ inline auto m_c_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) { return m_c_Select(Op0, m_True(), Op1); } +inline auto m_CanonicalIV() { return class_match(); } + template using VPScalarIVSteps_match = Recipe_match, 0, false, VPScalarIVStepsRecipe>; diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index 7c5d44daf003f..ef6ac493b7132 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -72,8 +72,10 @@ struct VPlanTransforms { dbgs() << Plan << '\n'; } #endif - if (VerifyEachVPlan && EnableVerify) - verifyVPlanIsValid(Plan); + if (VerifyEachVPlan && EnableVerify) { + if (!verifyVPlanIsValid(Plan)) + report_fatal_error("Broken VPlan found, compilation aborted!"); + } }}; return std::forward(Pass)(Plan, std::forward(Args)...); diff --git a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp index 35518c069bb3c..d62a71883b6ea 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp @@ -139,6 +139,27 @@ bool VPlanVerifier::verifyPhiRecipes(const VPBasicBlock *VPBB) { return true; } +static bool isKnownMonotonic(VPValue *V) { + VPValue *X, *Y; + // TODO: Check for hasNoUnsignedWrap() when we set nuw in VPlanUnroll + if (match(V, m_Add(m_VPValue(X), m_VPValue(Y)))) + return isKnownMonotonic(X) && isKnownMonotonic(Y); + if (match(V, m_StepVector())) + return true; + // Only handle a subset of IVs until we can guarantee there's no overflow. + if (auto *WidenIV = dyn_cast(V)) + return WidenIV->isCanonical(); + if (auto *Steps = dyn_cast(V)) + return match(Steps->getOperand(0), + m_CombineOr( + m_CanonicalIV(), + m_DerivedIV(m_ZeroInt(), m_CanonicalIV(), m_One()))) && + match(Steps->getStepValue(), m_One()); + if (isa(V)) + return true; + return vputils::isUniformAcrossVFsAndUFs(V); +} + bool VPlanVerifier::verifyLastActiveLaneRecipe( const VPInstruction &LastActiveLane) const { assert(LastActiveLane.getOpcode() == VPInstruction::LastActiveLane && @@ -150,18 +171,19 @@ bool VPlanVerifier::verifyLastActiveLaneRecipe( } const VPlan &Plan = *LastActiveLane.getParent()->getPlan(); - // All operands must be prefix-mask. Currently we check for header masks or - // EVL-derived masks, as those are currently the only operands in practice, - // but this may need updating in the future. + // All operands must be prefix-mask. This means an icmp ult/ule LHS, RHS where + // the LHS is monotonically increasing and RHS is uniform across VFs and UF. for (VPValue *Op : LastActiveLane.operands()) { if (vputils::isHeaderMask(Op, Plan)) continue; - // Masks derived from EVL are also fine. - auto BroadcastOrEVL = - m_CombineOr(m_Broadcast(m_EVL(m_VPValue())), m_EVL(m_VPValue())); - if (match(Op, m_CombineOr(m_ICmp(m_StepVector(), BroadcastOrEVL), - m_ICmp(BroadcastOrEVL, m_StepVector())))) + CmpPredicate Pred; + VPValue *LHS, *RHS; + if (match(Op, m_ICmp(Pred, m_VPValue(LHS), m_VPValue(RHS))) && + (Pred == CmpInst::ICMP_ULE || Pred == CmpInst::ICMP_ULT) && + isKnownMonotonic(LHS) && + (vputils::isUniformAcrossVFsAndUFs(RHS) || + match(RHS, m_EVL(m_VPValue())))) continue; errs() << "LastActiveLane operand "; diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp index f07ee94105ff8..2472e18b654e2 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp @@ -16,6 +16,10 @@ using namespace llvm; +namespace llvm { +LLVM_ABI extern cl::opt VerifyEachVPlan; +} // namespace llvm + using VPVerifierTest = VPlanTestBase; namespace { @@ -342,6 +346,28 @@ TEST_F(VPVerifierTest, NonHeaderPHIInHeader) { delete PHINode; } +TEST_F(VPVerifierTest, testRUN_VPLAN_PASS) { + VPlan &Plan = getPlan(); + VPIRValue *Zero = Plan.getConstantInt(32, 0); + VPInstruction *DefI = + new VPInstruction(Instruction::Add, {Zero, Zero}, + VPIRFlags::getDefaultFlags(Instruction::Add)); + VPInstruction *UseI = + new VPInstruction(Instruction::Sub, {DefI, Zero}, + VPIRFlags::getDefaultFlags(Instruction::Sub)); + VPBasicBlock *VPBB1 = Plan.getEntry(); + VPBB1->appendRecipe(UseI); + VPBB1->appendRecipe(DefI); + + bool OrigVerifyEachVPlan = VerifyEachVPlan; + VerifyEachVPlan = true; + llvm::scope_exit _([&]() { VerifyEachVPlan = OrigVerifyEachVPlan; }); + auto NopPass = [](VPlan &Plan) {}; + EXPECT_DEATH( + { VPlanTransforms::runPass("simplifyRecipes", NopPass, Plan); }, + "Broken VPlan found, compilation aborted!"); +} + class VPIRVerifierTest : public VPlanTestIRBase {}; TEST_F(VPIRVerifierTest, testVerifyIRPhiInScalarHeaderVPIRBB) {