[VPlan] Assert vplan-verify-each result and fix LastActiveLane verification

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -364,7 +364,7 @@ cl::opt<bool>
                           cl::init(false),
 #endif
                           cl::Hidden,
-                          cl::desc("Verfiy VPlans after VPlan transforms."));
+                          cl::desc("Verify VPlans after VPlan transforms."));
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 cl::opt<bool> llvm::VPlanPrintAfterAll(

llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h

@@ -859,6 +859,8 @@ inline auto m_c_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) {
   return m_c_Select(Op0, m_True(), Op1);
 }
 
+inline auto m_CanonicalIV() { return class_match<VPCanonicalIVPHIRecipe>(); }
+
 template <typename Op0_t, typename Op1_t, typename Op2_t>
 using VPScalarIVSteps_match = Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, 0,
                                            false, VPScalarIVStepsRecipe>;

llvm/lib/Transforms/Vectorize/VPlanTransforms.h

@@ -72,8 +72,10 @@ struct VPlanTransforms {
           dbgs() << Plan << '\n';
       }
 #endif
-      if (VerifyEachVPlan && EnableVerify)
-        verifyVPlanIsValid(Plan);
+      if (VerifyEachVPlan && EnableVerify) {
+        if (!verifyVPlanIsValid(Plan))
+          report_fatal_error("Broken VPlan found, compilation aborted!");
+      }
     }};
 
     return std::forward<PassTy>(Pass)(Plan, std::forward<ArgsTy>(Args)...);

llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp

@@ -139,6 +139,27 @@ bool VPlanVerifier::verifyPhiRecipes(const VPBasicBlock *VPBB) {
   return true;
 }
 
+static bool isKnownMonotonic(VPValue *V) {
+  VPValue *X, *Y;
+  // TODO: Check for hasNoUnsignedWrap() when we set nuw in VPlanUnroll
+  if (match(V, m_Add(m_VPValue(X), m_VPValue(Y))))
+    return isKnownMonotonic(X) && isKnownMonotonic(Y);
+  if (match(V, m_StepVector()))
+    return true;
+  // Only handle a subset of IVs until we can guarantee there's no overflow.
+  if (auto *WidenIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(V))
+    return WidenIV->isCanonical();
+  if (auto *Steps = dyn_cast<VPScalarIVStepsRecipe>(V))
+    return match(Steps->getOperand(0),
+                 m_CombineOr(
+                     m_CanonicalIV(),
+                     m_DerivedIV(m_ZeroInt(), m_CanonicalIV(), m_One()))) &&
+           match(Steps->getStepValue(), m_One());
+  if (isa<VPWidenCanonicalIVRecipe>(V))
+    return true;
+  return vputils::isUniformAcrossVFsAndUFs(V);
+}
+
 bool VPlanVerifier::verifyLastActiveLaneRecipe(
     const VPInstruction &LastActiveLane) const {
   assert(LastActiveLane.getOpcode() == VPInstruction::LastActiveLane &&
@@ -150,18 +171,19 @@ bool VPlanVerifier::verifyLastActiveLaneRecipe(
   }
 
   const VPlan &Plan = *LastActiveLane.getParent()->getPlan();
-  // All operands must be prefix-mask. Currently we check for header masks or
-  // EVL-derived masks, as those are currently the only operands in practice,
-  // but this may need updating in the future.
+  // All operands must be prefix-mask. This means an icmp ult/ule LHS, RHS where
+  // the LHS is monotonically increasing and RHS is uniform across VFs and UF.
   for (VPValue *Op : LastActiveLane.operands()) {
     if (vputils::isHeaderMask(Op, Plan))
       continue;
 
-    // Masks derived from EVL are also fine.
-    auto BroadcastOrEVL =
-        m_CombineOr(m_Broadcast(m_EVL(m_VPValue())), m_EVL(m_VPValue()));
-    if (match(Op, m_CombineOr(m_ICmp(m_StepVector(), BroadcastOrEVL),
-                              m_ICmp(BroadcastOrEVL, m_StepVector()))))
+    CmpPredicate Pred;
+    VPValue *LHS, *RHS;
+    if (match(Op, m_ICmp(Pred, m_VPValue(LHS), m_VPValue(RHS))) &&
+        (Pred == CmpInst::ICMP_ULE || Pred == CmpInst::ICMP_ULT) &&
+        isKnownMonotonic(LHS) &&
+        (vputils::isUniformAcrossVFsAndUFs(RHS) ||
+         match(RHS, m_EVL(m_VPValue()))))
       continue;
 
     errs() << "LastActiveLane operand ";

llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp

@@ -16,6 +16,10 @@
 
 using namespace llvm;
 
+namespace llvm {
+LLVM_ABI extern cl::opt<bool> VerifyEachVPlan;
+} // namespace llvm
+
 using VPVerifierTest = VPlanTestBase;
 
 namespace {
@@ -342,6 +346,28 @@ TEST_F(VPVerifierTest, NonHeaderPHIInHeader) {
   delete PHINode;
 }
 
+TEST_F(VPVerifierTest, testRUN_VPLAN_PASS) {
+  VPlan &Plan = getPlan();
+  VPIRValue *Zero = Plan.getConstantInt(32, 0);
+  VPInstruction *DefI =
+      new VPInstruction(Instruction::Add, {Zero, Zero},
+                        VPIRFlags::getDefaultFlags(Instruction::Add));
+  VPInstruction *UseI =
+      new VPInstruction(Instruction::Sub, {DefI, Zero},
+                        VPIRFlags::getDefaultFlags(Instruction::Sub));
+  VPBasicBlock *VPBB1 = Plan.getEntry();
+  VPBB1->appendRecipe(UseI);
+  VPBB1->appendRecipe(DefI);
+
+  bool OrigVerifyEachVPlan = VerifyEachVPlan;
+  VerifyEachVPlan = true;
+  llvm::scope_exit _([&]() { VerifyEachVPlan = OrigVerifyEachVPlan; });
+  auto NopPass = [](VPlan &Plan) {};
+  EXPECT_DEATH(
+      { VPlanTransforms::runPass("simplifyRecipes", NopPass, Plan); },
+      "Broken VPlan found, compilation aborted!");
+}
+
 class VPIRVerifierTest : public VPlanTestIRBase {};
 
 TEST_F(VPIRVerifierTest, testVerifyIRPhiInScalarHeaderVPIRBB) {