Skip to content

[LV] Simplify and unify resume value handling for epilogue vec. #185969

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fhahn merged 9 commits into from
Mar 16, 2026
Merged

[LV] Simplify and unify resume value handling for epilogue vec. #185969

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
90ca940
[LV] Simplify and unify resume value handling for epilogue vec.
fhahn Mar 9, 2026
539884b
Merge remote-tracking branch 'origin/main' into lv-simplify-epi-scala…
fhahn Mar 12, 2026
03b6f41
!fixup address latest commits, thanks
fhahn Mar 12, 2026
83fbf86
!fixup address latest comments, thanks
fhahn Mar 12, 2026
995c318
!fixup remove dead code
fhahn Mar 12, 2026
9c3b914
Merge remote-tracking branch 'origin/main' into lv-simplify-epi-scala…
fhahn Mar 13, 2026
f505bd5
!fixup address latest comments, thanks
fhahn Mar 13, 2026
f98ca63
Merge remote-tracking branch 'origin/main' into lv-simplify-epi-scala…
fhahn Mar 15, 2026
8bb9ff6
Merge remote-tracking branch 'origin/main' into lv-simplify-epi-scala…
fhahn Mar 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
242 changes: 50 additions & 192 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2423,20 +2423,6 @@ BasicBlock *InnerLoopVectorizer::createScalarPreheader(StringRef Prefix) {
Twine(Prefix) + "scalar.ph");
}

/// Return the expanded step for \p ID using \p ExpandedSCEVs to look up SCEV
/// expansion results.
static Value *getExpandedStep(const InductionDescriptor &ID,
const SCEV2ValueTy &ExpandedSCEVs) {
const SCEV *Step = ID.getStep();
if (auto *C = dyn_cast<SCEVConstant>(Step))
return C->getValue();
if (auto *U = dyn_cast<SCEVUnknown>(Step))
return U->getValue();
Value *V = ExpandedSCEVs.lookup(Step);
assert(V && "SCEV must be expanded at this point");
return V;
}

/// Knowing that loop \p L executes a single vector iteration, add instructions
/// that will get simplified and thus should not have any cost to \p
/// InstsToIgnore.
Expand Down Expand Up @@ -7342,73 +7328,6 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() {
return BestFactor;
}

// If \p EpiResumePhiR is resume VPPhi for a reduction when vectorizing the
// epilog loop, fix the reduction's scalar PHI node by adding the incoming value
// from the main vector loop.
static void fixReductionScalarResumeWhenVectorizingEpilog(
VPPhi *EpiResumePhiR, PHINode &EpiResumePhi, BasicBlock *BypassBlock) {
using namespace VPlanPatternMatch;
// Get the VPInstruction computing the reduction result in the middle block.
// The first operand may not be from the middle block if it is not connected
// to the scalar preheader. In that case, there's nothing to fix.
VPValue *Incoming = EpiResumePhiR->getOperand(0);
match(Incoming, VPlanPatternMatch::m_ZExtOrSExt(
VPlanPatternMatch::m_VPValue(Incoming)));
auto *EpiRedResult = dyn_cast<VPInstruction>(Incoming);
if (!EpiRedResult)
return;

VPValue *BackedgeVal;
bool IsFindIV = false;
if (EpiRedResult->getOpcode() == VPInstruction::ComputeAnyOfResult ||
EpiRedResult->getOpcode() == VPInstruction::ComputeReductionResult)
BackedgeVal = EpiRedResult->getOperand(EpiRedResult->getNumOperands() - 1);
else if (matchFindIVResult(EpiRedResult, m_VPValue(BackedgeVal), m_VPValue()))
IsFindIV = true;
else
return;

auto *EpiRedHeaderPhi = cast_if_present<VPReductionPHIRecipe>(
vputils::findRecipe(BackedgeVal, IsaPred<VPReductionPHIRecipe>));
if (!EpiRedHeaderPhi) {
match(BackedgeVal,
VPlanPatternMatch::m_Select(VPlanPatternMatch::m_VPValue(),
VPlanPatternMatch::m_VPValue(BackedgeVal),
VPlanPatternMatch::m_VPValue()));
EpiRedHeaderPhi = cast<VPReductionPHIRecipe>(
vputils::findRecipe(BackedgeVal, IsaPred<VPReductionPHIRecipe>));
}

Value *MainResumeValue;
if (auto *VPI = dyn_cast<VPInstruction>(EpiRedHeaderPhi->getStartValue())) {
assert((VPI->getOpcode() == VPInstruction::Broadcast ||
VPI->getOpcode() == VPInstruction::ReductionStartVector) &&
"unexpected start recipe");
MainResumeValue = VPI->getOperand(0)->getUnderlyingValue();
} else
MainResumeValue = EpiRedHeaderPhi->getStartValue()->getUnderlyingValue();
if (EpiRedResult->getOpcode() == VPInstruction::ComputeAnyOfResult) {
[[maybe_unused]] Value *StartV =
EpiRedResult->getOperand(0)->getLiveInIRValue();
auto *Cmp = cast<ICmpInst>(MainResumeValue);
assert(Cmp->getPredicate() == CmpInst::ICMP_NE &&
"AnyOf expected to start with ICMP_NE");
assert(Cmp->getOperand(1) == StartV &&
"AnyOf expected to start by comparing main resume value to original "
"start value");
MainResumeValue = Cmp->getOperand(0);
} else if (IsFindIV) {
MainResumeValue = cast<SelectInst>(MainResumeValue)->getFalseValue();
}
PHINode *MainResumePhi = cast<PHINode>(MainResumeValue);

// When fixing reductions in the epilogue loop we should already have
// created a bc.merge.rdx Phi after the main vector body. Ensure that we carry
// over the incoming values correctly.
EpiResumePhi.setIncomingValueForBlock(
BypassBlock, MainResumePhi->getIncomingValueForBlock(BypassBlock));
}

DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
InnerLoopVectorizer &ILV, DominatorTree *DT, bool VectorizingEpilogue) {
Expand Down Expand Up @@ -8992,35 +8911,9 @@ LoopVectorizePass::LoopVectorizePass(LoopVectorizeOptions Opts)
!EnableLoopVectorization) {}

/// Prepare \p MainPlan for vectorizing the main vector loop during epilogue
/// vectorization. Remove ResumePhis from \p MainPlan for inductions that
/// don't have a corresponding wide induction in \p EpiPlan.
static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
// Collect PHI nodes of widened phis in the VPlan for the epilogue. Those
// will need their resume-values computed in the main vector loop. Others
// can be removed from the main VPlan.
SmallPtrSet<PHINode *, 2> EpiWidenedPhis;
for (VPRecipeBase &R :
EpiPlan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
if (isa<VPCanonicalIVPHIRecipe>(&R))
continue;
EpiWidenedPhis.insert(
cast<PHINode>(R.getVPSingleValue()->getUnderlyingValue()));
}
for (VPRecipeBase &R :
make_early_inc_range(MainPlan.getScalarHeader()->phis())) {
auto *VPIRInst = cast<VPIRPhi>(&R);
if (EpiWidenedPhis.contains(&VPIRInst->getIRPhi()))
continue;
// There is no corresponding wide induction in the epilogue plan that would
// need a resume value. Remove the VPIRInst wrapping the scalar header phi
// together with the corresponding ResumePhi. The resume values for the
// scalar loop will be created during execution of EpiPlan.
VPRecipeBase *ResumePhi = VPIRInst->getOperand(0)->getDefiningRecipe();
VPIRInst->eraseFromParent();
ResumePhi->eraseFromParent();
}
RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, MainPlan);

/// vectorization.
static SmallVector<VPInstruction *>
preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
Comment on lines +8914 to +8915

@artagnon artagnon Mar 12, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it possible to make preparePlanFor(Main|Epilogue)VectorLoop totally uniform, and return a SmallVector of VPInstruction resume values or InstsToMove in both?

@fhahn fhahn Mar 12, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not sure exactly, their returns are very different, but perhaps I am missing a simplification you are thinking of?

using namespace VPlanPatternMatch;
// When vectorizing the epilogue, FindFirstIV & FindLastIV reductions can
// introduce multiple uses of undef/poison. If the reduction start value may
Expand Down Expand Up @@ -9077,14 +8970,26 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
{}, "vec.epilog.resume.val");
} else {
ResumePhi = cast<VPPhi>(&*ResumePhiIter);
if (MainScalarPH->begin() == MainScalarPH->end())
ResumePhi->moveBefore(*MainScalarPH, MainScalarPH->end());
else if (&*MainScalarPH->begin() != ResumePhi)
ResumePhi->setName("vec.epilog.resume.val");
if (&MainScalarPH->front() != ResumePhi)
ResumePhi->moveBefore(*MainScalarPH, MainScalarPH->begin());
}
// Add a user to to make sure the resume phi won't get removed.
VPBuilder(MainScalarPH)
.createNaryOp(VPInstruction::ResumeForEpilogue, ResumePhi);

// Create a ResumeForEpilogue for the canonical IV resume as the
// first non-phi, to keep it alive for the epilogue.
VPBuilder ResumeBuilder(MainScalarPH);
ResumeBuilder.createNaryOp(VPInstruction::ResumeForEpilogue, ResumePhi);
Comment thread
artagnon marked this conversation as resolved.

// Create ResumeForEpilogue instructions corresponding resume phis for the
// VPIRPhis in the scalar header of the main plan and return them so they can
// be used as resume values when vectorizing the epilogue.
return to_vector(
map_range(MainPlan.getScalarHeader()->phis(), [&](VPRecipeBase &R) {
Comment thread
artagnon marked this conversation as resolved.
assert(isa<VPIRPhi>(R) &&
"only VPIRPhis expected in the scalar preheader");
return ResumeBuilder.createNaryOp(VPInstruction::ResumeForEpilogue,
R.getOperand(0));
Comment thread
artagnon marked this conversation as resolved.
}));
}

/// Prepare \p Plan for vectorizing the epilogue loop. That is, re-use expanded
Expand Down Expand Up @@ -9292,39 +9197,11 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
return InstsToMove;
}

// Generate bypass values from the additional bypass block. Note that when the
// vectorized epilogue is skipped due to iteration count check, then the
// resume value for the induction variable comes from the trip count of the
// main vector loop, passed as the second argument.
static Value *createInductionAdditionalBypassValues(
PHINode *OrigPhi, const InductionDescriptor &II, IRBuilder<> &BypassBuilder,
const SCEV2ValueTy &ExpandedSCEVs, Value *MainVectorTripCount,
Instruction *OldInduction) {
Value *Step = getExpandedStep(II, ExpandedSCEVs);
Comment thread
artagnon marked this conversation as resolved.
// For the primary induction the additional bypass end value is known.
// Otherwise it is computed.
Value *EndValueFromAdditionalBypass = MainVectorTripCount;
if (OrigPhi != OldInduction) {
auto *BinOp = II.getInductionBinOp();
// Fast-math-flags propagate from the original induction instruction.
if (isa_and_nonnull<FPMathOperator>(BinOp))
BypassBuilder.setFastMathFlags(BinOp->getFastMathFlags());

// Compute the end value for the additional bypass.
EndValueFromAdditionalBypass =
emitTransformedIndex(BypassBuilder, MainVectorTripCount,
II.getStartValue(), Step, II.getKind(), BinOp);
Comment thread
artagnon marked this conversation as resolved.
EndValueFromAdditionalBypass->setName("ind.end");
}
return EndValueFromAdditionalBypass;
}

static void fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L,
VPlan &BestEpiPlan,
LoopVectorizationLegality &LVL,
const SCEV2ValueTy &ExpandedSCEVs,
Value *MainVectorTripCount) {
// Fix reduction resume values from the additional bypass block.
static void
fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L,
VPlan &BestEpiPlan,
ArrayRef<VPInstruction *> ResumeValues) {
// Fix resume values from the additional bypass block.
BasicBlock *PH = L->getLoopPreheader();
for (auto *Pred : predecessors(PH)) {
for (PHINode &Phi : PH->phis()) {
Expand All @@ -9335,40 +9212,13 @@ static void fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L,
}
auto *ScalarPH = cast<VPIRBasicBlock>(BestEpiPlan.getScalarPreheader());
if (ScalarPH->hasPredecessors()) {
// If ScalarPH has predecessors, we may need to update its reduction
// resume values.
for (const auto &[R, IRPhi] :
zip(ScalarPH->phis(), ScalarPH->getIRBasicBlock()->phis())) {
fixReductionScalarResumeWhenVectorizingEpilog(cast<VPPhi>(&R), IRPhi,
BypassBlock);
}
}

// Fix induction resume values from the additional bypass block.
IRBuilder<> BypassBuilder(BypassBlock, BypassBlock->getFirstInsertionPt());
for (const auto &[IVPhi, II] : LVL.getInductionVars()) {
Value *V = createInductionAdditionalBypassValues(
IVPhi, II, BypassBuilder, ExpandedSCEVs, MainVectorTripCount,
LVL.getPrimaryInduction());
// TODO: Directly add as extra operand to the VPResumePHI recipe.
if (auto *Inc = dyn_cast<PHINode>(IVPhi->getIncomingValueForBlock(PH))) {
if (Inc->getBasicBlockIndex(BypassBlock) != -1)
Inc->setIncomingValueForBlock(BypassBlock, V);
} else {
// If the resume value in the scalar preheader was simplified (e.g., when
// narrowInterleaveGroups optimized away the resume PHIs), create a new
// PHI to merge the bypass value with the original value.
Value *OrigVal = IVPhi->getIncomingValueForBlock(PH);
PHINode *NewPhi =
PHINode::Create(IVPhi->getType(), pred_size(PH), "bc.resume.val",
PH->getFirstNonPHIIt());
for (auto *Pred : predecessors(PH)) {
if (Pred == BypassBlock)
NewPhi->addIncoming(V, Pred);
else
NewPhi->addIncoming(OrigVal, Pred);
}
IVPhi->setIncomingValueForBlock(PH, NewPhi);
// Fix resume values for inductions and reductions from the additional
// bypass block using the incoming values from the main loop's resume phis.
for (auto [ResumeV, IRPhi] :
zip(ResumeValues, ScalarPH->getIRBasicBlock()->phis())) {
auto *MainResumePhi = cast<PHINode>(ResumeV->getUnderlyingValue());
IRPhi.setIncomingValueForBlock(
BypassBlock, MainResumePhi->getIncomingValueForBlock(BypassBlock));
}
Comment thread
artagnon marked this conversation as resolved.
Outdated
}
}
Expand All @@ -9378,11 +9228,12 @@ static void fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L,
// and runtime checks of the main loop, as well as updating various phis. \p
// InstsToMove contains instructions that need to be moved to the preheader of
// the epilogue vector loop.
static void connectEpilogueVectorLoop(
VPlan &EpiPlan, Loop *L, EpilogueLoopVectorizationInfo &EPI,
DominatorTree *DT, LoopVectorizationLegality &LVL,
DenseMap<const SCEV *, Value *> &ExpandedSCEVs, GeneratedRTChecks &Checks,
ArrayRef<Instruction *> InstsToMove) {
static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
EpilogueLoopVectorizationInfo &EPI,
DominatorTree *DT,
GeneratedRTChecks &Checks,
ArrayRef<Instruction *> InstsToMove,
ArrayRef<VPInstruction *> ResumeValues) {
BasicBlock *VecEpilogueIterationCountCheck =
Comment thread
artagnon marked this conversation as resolved.
cast<VPIRBasicBlock>(EpiPlan.getEntry())->getIRBasicBlock();

Expand Down Expand Up @@ -9465,7 +9316,13 @@ static void connectEpilogueVectorLoop(
// after executing the main loop. We need to update the resume values of
// inductions and reductions during epilogue vectorization.
fixScalarResumeValuesFromBypass(VecEpilogueIterationCountCheck, L, EpiPlan,
LVL, ExpandedSCEVs, EPI.VectorTripCount);
ResumeValues);

// Remove dead phis that were moved to the epilogue preheader but are unused
// (e.g., resume phis for inductions not widened in the epilogue vector loop).
for (PHINode &Phi : make_early_inc_range(VecEpiloguePreHeader->phis()))
if (Phi.use_empty())
Phi.eraseFromParent();
Comment thread
artagnon marked this conversation as resolved.
}

bool LoopVectorizePass::processLoop(Loop *L) {
Expand Down Expand Up @@ -9852,7 +9709,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
VPlan &BestEpiPlan = LVP.getPlanFor(EpilogueVF.Width);
BestEpiPlan.getMiddleBlock()->setName("vec.epilog.middle.block");
BestEpiPlan.getVectorPreheader()->setName("vec.epilog.ph");
preparePlanForMainVectorLoop(*BestMainPlan, BestEpiPlan);
SmallVector<VPInstruction *> ResumeValues =
preparePlanForMainVectorLoop(*BestMainPlan, BestEpiPlan);
EpilogueLoopVectorizationInfo EPI(VF.Width, IC, EpilogueVF.Width, 1,
BestEpiPlan);
EpilogueVectorizerMainLoop MainILV(L, PSE, LI, DT, TTI, AC, EPI, &CM,
Expand All @@ -9869,8 +9727,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
BestEpiPlan, L, ExpandedSCEVs, EPI, CM, *PSE.getSE());
LVP.executePlan(EPI.EpilogueVF, EPI.EpilogueUF, BestEpiPlan, EpilogILV, DT,
true);
connectEpilogueVectorLoop(BestEpiPlan, L, EPI, DT, LVL, ExpandedSCEVs,
Checks, InstsToMove);
connectEpilogueVectorLoop(BestEpiPlan, L, EPI, DT, Checks, InstsToMove,
ResumeValues);
++LoopsEpilogueVectorized;
} else {
InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, IC, &CM, Checks,
Expand Down
6 changes: 6 additions & 0 deletions llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1321,6 +1321,12 @@ void VPInstruction::execute(VPTransformState &State) {
"scalar value but not only first lane defined");
State.set(this, GeneratedValue,
/*IsScalar*/ GeneratesPerFirstLaneOnly);
if (getOpcode() == VPInstruction::ResumeForEpilogue) {
// FIXME: This is a workaround to enable reliable updates of the scalar loop
// resume phis, when vectorizing the epilogue. Must be removed once epilogue
// vectorization explicitly connects VPlans.
setUnderlyingValue(GeneratedValue);
}
}

bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
Expand Down
3 changes: 1 addition & 2 deletions llvm/test/Transforms/LoopVectorize/AArch64/epilog-iv-select-cmp.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ define i8 @select_icmp_var_start(ptr %a, i8 %n, i8 %start) {
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
; CHECK-NEXT: [[IND_END:%.*]] = trunc i32 [[N_VEC]] to i8
; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i32 [[N_MOD_VF]], 8
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
; CHECK: [[VEC_EPILOG_PH]]:
Expand Down Expand Up @@ -84,7 +83,7 @@ define i8 @select_icmp_var_start(ptr %a, i8 %n, i8 %start) {
; CHECK-NEXT: [[CMP_N16:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC5]]
; CHECK-NEXT: br i1 [[CMP_N16]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL17:%.*]] = phi i8 [ [[TMP16]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; CHECK-NEXT: [[BC_RESUME_VAL17:%.*]] = phi i8 [ [[TMP16]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP3]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX18:%.*]] = phi i8 [ [[RDX_SELECT15]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], %[[ITER_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
Expand Down
4 changes: 2 additions & 2 deletions llvm/test/Transforms/LoopVectorize/AArch64/epilog-vectorization-factors.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -446,6 +446,8 @@ define void @trip_count_based_on_ptrtoint(i64 %x) "target-cpu"="apple-m1" {
; CHECK: vector.ph:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 16
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[N_VEC]], 4
; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[TMP12]]
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
Expand All @@ -465,8 +467,6 @@ define void @trip_count_based_on_ptrtoint(i64 %x) "target-cpu"="apple-m1" {
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[N_VEC]], 4
; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[PTR_START]], i64 [[TMP12]]
; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF11]]
; CHECK: vec.epilog.ph:
Expand Down
Loading