From 68eb4fc80e82cfd6fd970bf5077013db078a35c8 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Thu, 8 Jan 2026 16:27:40 +0800 Subject: [PATCH 1/9] [VPlan] Split out optimizeEVLMasks. NFC Addresses part of #153144 and splits off part of #166164 There are two parts to the EVL transform: 1) Convert the loop so the number of elements processed each iteration is EVL, not VF. The IV and header mask are replaced with EVL-based variants. 2) Optimize users of the EVL based header mask to VP intrinsic based recipes. 1) changes the semantics of the vector loop region, whereas the 2) needs to preserve it. This splits 2) out so we don't mix the two up, and allows us to move 1) earlier in the pipeline in a future PR. --- .../Transforms/Vectorize/LoopVectorize.cpp | 6 +- .../Transforms/Vectorize/VPlanTransforms.cpp | 82 +++++++++---------- .../Transforms/Vectorize/VPlanTransforms.h | 8 ++ 3 files changed, 49 insertions(+), 47 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 31c44fc46c973..903fb7b25ad6a 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8328,10 +8328,12 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF, VPlanTransforms::runPass(VPlanTransforms::truncateToMinimalBitwidths, *Plan, CM.getMinimalBitwidths()); VPlanTransforms::runPass(VPlanTransforms::optimize, *Plan); - // TODO: try to put it close to addActiveLaneMask(). - if (CM.foldTailWithEVL()) + // TODO: try to put addExplicitVectorLength close to addActiveLaneMask + if (CM.foldTailWithEVL()) { VPlanTransforms::runPass(VPlanTransforms::addExplicitVectorLength, *Plan, CM.getMaxSafeElements()); + VPlanTransforms::runPass(VPlanTransforms::optimizeEVLMasks, *Plan); + } assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid"); VPlans.push_back(std::move(Plan)); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index b1880517a4199..e1dba4cf4a064 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2979,8 +2979,42 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask, return nullptr; } -/// Replace recipes with their EVL variants. -static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { +/// Optimize away any EVL-based header masks to VP intrinsic based recipes. +/// The transforms here need to preserve the original semantics. +void VPlanTransforms::optimizeEVLMasks(VPlan &Plan) { + // Find the EVL-based header mask if it exists: icmp ult step-vector, EVL + VPInstruction *HeaderMask = nullptr; + for (VPRecipeBase &R : *Plan.getVectorLoopRegion()->getEntryBasicBlock()) { + if (match(&R, m_ICmp(m_VPInstruction(), + m_EVL(m_VPValue())))) { + HeaderMask = cast(&R); + break; + } + } + if (!HeaderMask) + return; + + VPTypeAnalysis TypeInfo(Plan); + VPValue *EVL = HeaderMask->getOperand(1); + + for (VPUser *U : collectUsersRecursively(HeaderMask)) { + VPRecipeBase *R = cast(U); + if (auto *NewR = optimizeMaskToEVL(HeaderMask, *R, TypeInfo, *EVL)) { + NewR->insertBefore(R); + for (auto [Old, New] : + zip_equal(R->definedValues(), NewR->definedValues())) + Old->replaceAllUsesWith(New); + // Erase dead stores, the rest will be removed by removeDeadRecipes. + if (R->getNumDefinedValues() == 0) + R->eraseFromParent(); + } + } + removeDeadRecipes(Plan); +} + +/// After replacing the IV with a EVL-based IV, fixup recipes that use VF to use +/// the EVL instead to avoid incorrect updates on the penultimate iteration. +static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) { VPTypeAnalysis TypeInfo(Plan); VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion(); VPBasicBlock *Header = LoopRegion->getEntryBasicBlock(); @@ -3008,10 +3042,6 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { return isa(U); }); - // Defer erasing recipes till the end so that we don't invalidate the - // VPTypeAnalysis cache. - SmallVector ToErase; - // Create a scalar phi to track the previous EVL if fixed-order recurrence is // contained. bool ContainsFORs = @@ -3046,7 +3076,6 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { R.getDebugLoc()); VPSplice->insertBefore(&R); R.getVPSingleValue()->replaceAllUsesWith(VPSplice); - ToErase.push_back(&R); } } } @@ -3067,43 +3096,6 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { CmpInst::ICMP_ULT, Builder.createNaryOp(VPInstruction::StepVector, {}, EVLType), &EVL); HeaderMask->replaceAllUsesWith(EVLMask); - ToErase.push_back(HeaderMask->getDefiningRecipe()); - - // Try to optimize header mask recipes away to their EVL variants. - // TODO: Split optimizeMaskToEVL out and move into - // VPlanTransforms::optimize. transformRecipestoEVLRecipes should be run in - // tryToBuildVPlanWithVPRecipes beforehand. - for (VPUser *U : collectUsersRecursively(EVLMask)) { - auto *CurRecipe = cast(U); - VPRecipeBase *EVLRecipe = - optimizeMaskToEVL(EVLMask, *CurRecipe, TypeInfo, EVL); - if (!EVLRecipe) - continue; - - unsigned NumDefVal = EVLRecipe->getNumDefinedValues(); - assert(NumDefVal == CurRecipe->getNumDefinedValues() && - "New recipe must define the same number of values as the " - "original."); - EVLRecipe->insertBefore(CurRecipe); - if (isa( - EVLRecipe)) { - for (unsigned I = 0; I < NumDefVal; ++I) { - VPValue *CurVPV = CurRecipe->getVPValue(I); - CurVPV->replaceAllUsesWith(EVLRecipe->getVPValue(I)); - } - } - ToErase.push_back(CurRecipe); - } - // Remove dead EVL mask. - if (EVLMask->getNumUsers() == 0) - ToErase.push_back(EVLMask->getDefiningRecipe()); - - for (VPRecipeBase *R : reverse(ToErase)) { - SmallVector PossiblyDead(R->operands()); - R->eraseFromParent(); - for (VPValue *Op : PossiblyDead) - recursivelyDeleteDeadRecipes(Op); - } } /// Add a VPEVLBasedIVPHIRecipe and related recipes to \p Plan and @@ -3201,7 +3193,7 @@ void VPlanTransforms::addExplicitVectorLength( DebugLoc::getCompilerGenerated(), "avl.next"); AVLPhi->addOperand(NextAVL); - transformRecipestoEVLRecipes(Plan, *VPEVL); + fixupVFUsersForEVL(Plan, *VPEVL); // Replace all uses of VPCanonicalIVPHIRecipe by // VPEVLBasedIVPHIRecipe except for the canonical IV increment. diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index f4ecee4e7f04f..35af0a76019dc 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -266,6 +266,14 @@ struct VPlanTransforms { addExplicitVectorLength(VPlan &Plan, const std::optional &MaxEVLSafeElements); + /// Optimize recipes which use an EVL based header mask to a VP intrinsic: + /// + /// %mask = icmp step-vector, EVL + /// %load = load %ptr, %mask + /// --> + /// %load = vp.load %ptr, EVL + static void optimizeEVLMasks(VPlan &Plan); + // For each Interleave Group in \p InterleaveGroups replace the Recipes // widening its memory instructions with a single VPInterleaveRecipe at its // insertion point. From abbb9390d419fc0089cf282776cd8f38da40e817 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Thu, 8 Jan 2026 16:34:14 +0800 Subject: [PATCH 2/9] Remove dead recipes in addExplicitVectorLength too --- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index e1dba4cf4a064..b4ba6b2ee2644 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -3194,6 +3194,7 @@ void VPlanTransforms::addExplicitVectorLength( AVLPhi->addOperand(NextAVL); fixupVFUsersForEVL(Plan, *VPEVL); + removeDeadRecipes(Plan); // Replace all uses of VPCanonicalIVPHIRecipe by // VPEVLBasedIVPHIRecipe except for the canonical IV increment. From e5dbe6694c165b5a4d1a891ca4e66bb87808592b Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Sat, 10 Jan 2026 15:30:35 +0800 Subject: [PATCH 3/9] Use m_SpecificICmp, use m_StepVector, pattern match EVL --- llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h | 5 +++++ llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 10 +++++----- llvm/lib/Transforms/Vectorize/VPlanTransforms.h | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h index 70b174509fc47..0626bb1077f0e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h +++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h @@ -633,6 +633,11 @@ struct SpecificCmp_match { : Predicate(Pred), Op0(LHS), Op1(RHS) {} bool match(const VPValue *V) const { + auto *DefR = V->getDefiningRecipe(); + return DefR && match(DefR); + } + + bool match(const VPRecipeBase *V) const { CmpPredicate CurrentPred; return Cmp_match(CurrentPred, Op0, Op1) .match(V) && diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index b4ba6b2ee2644..9747820ee883f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2983,11 +2983,12 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask, /// The transforms here need to preserve the original semantics. void VPlanTransforms::optimizeEVLMasks(VPlan &Plan) { // Find the EVL-based header mask if it exists: icmp ult step-vector, EVL - VPInstruction *HeaderMask = nullptr; + VPValue *HeaderMask = nullptr, *EVL = nullptr; for (VPRecipeBase &R : *Plan.getVectorLoopRegion()->getEntryBasicBlock()) { - if (match(&R, m_ICmp(m_VPInstruction(), - m_EVL(m_VPValue())))) { - HeaderMask = cast(&R); + if (match(&R, m_SpecificICmp(CmpInst::ICMP_ULT, m_StepVector(), + m_VPValue(EVL))) && + match(EVL, m_EVL(m_VPValue()))) { + HeaderMask = R.getVPSingleValue(); break; } } @@ -2995,7 +2996,6 @@ void VPlanTransforms::optimizeEVLMasks(VPlan &Plan) { return; VPTypeAnalysis TypeInfo(Plan); - VPValue *EVL = HeaderMask->getOperand(1); for (VPUser *U : collectUsersRecursively(HeaderMask)) { VPRecipeBase *R = cast(U); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index 35af0a76019dc..8f3d3b40c1c77 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -268,7 +268,7 @@ struct VPlanTransforms { /// Optimize recipes which use an EVL based header mask to a VP intrinsic: /// - /// %mask = icmp step-vector, EVL + /// %mask = icmp ult step-vector, EVL /// %load = load %ptr, %mask /// --> /// %load = vp.load %ptr, EVL From 5a2b815c74856bb412de2bc69803b4124afdb855 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Sat, 10 Jan 2026 15:59:01 +0800 Subject: [PATCH 4/9] Erase recipes at the end to avoid invalidating TypeInfo --- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 9747820ee883f..4ee5dded29a83 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2608,7 +2608,6 @@ void VPlanTransforms::truncateToMinimalBitwidths( ProcessedIter->second = NewOp; R.setOperand(Idx, NewOp); } - } } } @@ -2996,6 +2995,7 @@ void VPlanTransforms::optimizeEVLMasks(VPlan &Plan) { return; VPTypeAnalysis TypeInfo(Plan); + SmallVector OldRecipes; for (VPUser *U : collectUsersRecursively(HeaderMask)) { VPRecipeBase *R = cast(U); @@ -3004,11 +3004,13 @@ void VPlanTransforms::optimizeEVLMasks(VPlan &Plan) { for (auto [Old, New] : zip_equal(R->definedValues(), NewR->definedValues())) Old->replaceAllUsesWith(New); - // Erase dead stores, the rest will be removed by removeDeadRecipes. - if (R->getNumDefinedValues() == 0) - R->eraseFromParent(); + OldRecipes.push_back(R); } } + // Erase recipes at the end so we don't invalidate TypeInfo. + for (VPRecipeBase *OldR : OldRecipes) + OldR->eraseFromParent(); + // Clean up any recipes now made dead. removeDeadRecipes(Plan); } From acf669c9b0eee7bb218121dcccd47c6088631b43 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Sat, 10 Jan 2026 16:12:03 +0800 Subject: [PATCH 5/9] Fix stray change --- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 4ee5dded29a83..62016bdffc502 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2608,6 +2608,7 @@ void VPlanTransforms::truncateToMinimalBitwidths( ProcessedIter->second = NewOp; R.setOperand(Idx, NewOp); } + } } } From 3a6438e3878daaa72bdab48169319cfc0b3a784d Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Sat, 10 Jan 2026 18:47:43 +0800 Subject: [PATCH 6/9] Address comments --- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 62016bdffc502..1ea057100e03b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -2997,7 +2997,6 @@ void VPlanTransforms::optimizeEVLMasks(VPlan &Plan) { VPTypeAnalysis TypeInfo(Plan); SmallVector OldRecipes; - for (VPUser *U : collectUsersRecursively(HeaderMask)) { VPRecipeBase *R = cast(U); if (auto *NewR = optimizeMaskToEVL(HeaderMask, *R, TypeInfo, *EVL)) { @@ -3011,7 +3010,6 @@ void VPlanTransforms::optimizeEVLMasks(VPlan &Plan) { // Erase recipes at the end so we don't invalidate TypeInfo. for (VPRecipeBase *OldR : OldRecipes) OldR->eraseFromParent(); - // Clean up any recipes now made dead. removeDeadRecipes(Plan); } From e550c42cafba7fdb26cf18c9bad3378f4572e82a Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Mon, 12 Jan 2026 14:55:32 +0800 Subject: [PATCH 7/9] Update comments --- .../Transforms/Vectorize/VPlanTransforms.cpp | 19 +++++++++++++++---- .../Transforms/Vectorize/VPlanTransforms.h | 3 ++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 1ea057100e03b..ab45e7a67b247 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -3007,14 +3007,16 @@ void VPlanTransforms::optimizeEVLMasks(VPlan &Plan) { OldRecipes.push_back(R); } } - // Erase recipes at the end so we don't invalidate TypeInfo. + // Erase the old recipes ourselves since removeDeadRecipes won't remove dead + // stores. Do this at the end so we don't invalidate TypeInfo. for (VPRecipeBase *OldR : OldRecipes) OldR->eraseFromParent(); removeDeadRecipes(Plan); } -/// After replacing the IV with a EVL-based IV, fixup recipes that use VF to use -/// the EVL instead to avoid incorrect updates on the penultimate iteration. +/// After replacing the canonical IV with a EVL-based IV, fixup recipes that use +/// VF to use the EVL instead to avoid incorrect updates on the penultimate +/// iteration. static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) { VPTypeAnalysis TypeInfo(Plan); VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion(); @@ -3099,11 +3101,20 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) { HeaderMask->replaceAllUsesWith(EVLMask); } -/// Add a VPEVLBasedIVPHIRecipe and related recipes to \p Plan and +/// Converts a tail folded vector loop region to step by @llvm.get.vector.length +/// elements instead of VF elements each iteration. +/// +/// - Add a VPEVLBasedIVPHIRecipe and related recipes to \p Plan and /// replaces all uses except the canonical IV increment of /// VPCanonicalIVPHIRecipe with a VPEVLBasedIVPHIRecipe. VPCanonicalIVPHIRecipe /// is used only for loop iterations counting after this transformation. /// +/// - The header mask is replaced with a header mask based on the EVL. +/// +/// - Plans with FORs have a new phi added to keep track of the EVL of the +/// previous iteration, and VPFirstOrderRecurrencePHIRecipes are replaced with +/// @llvm.vp.splice. +/// /// The function uses the following definitions: /// %StartV is the canonical induction start value. /// diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index 8f3d3b40c1c77..271c1035c18f3 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -266,7 +266,8 @@ struct VPlanTransforms { addExplicitVectorLength(VPlan &Plan, const std::optional &MaxEVLSafeElements); - /// Optimize recipes which use an EVL based header mask to a VP intrinsic: + /// Optimize recipes which use an EVL-based header mask to VP intrinsics, for + /// example: /// /// %mask = icmp ult step-vector, EVL /// %load = load %ptr, %mask From d307310d58edfcce1ba9a403b78f885ac26de21c Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 14 Jan 2026 14:21:14 +0800 Subject: [PATCH 8/9] Use recursivelyDeleteDeadRecipes --- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index ab45e7a67b247..b74a513cc128d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -3007,11 +3007,13 @@ void VPlanTransforms::optimizeEVLMasks(VPlan &Plan) { OldRecipes.push_back(R); } } - // Erase the old recipes ourselves since removeDeadRecipes won't remove dead - // stores. Do this at the end so we don't invalidate TypeInfo. - for (VPRecipeBase *OldR : OldRecipes) - OldR->eraseFromParent(); - removeDeadRecipes(Plan); + // Erase old recipes at the end so we don't invalidate TypeInfo. + for (VPRecipeBase *R : reverse(OldRecipes)) { + SmallVector PossiblyDead(R->operands()); + R->eraseFromParent(); + for (VPValue *Op : PossiblyDead) + recursivelyDeleteDeadRecipes(Op); + } } /// After replacing the canonical IV with a EVL-based IV, fixup recipes that use From 3d5108cec24fe754a33a3eeeae1d9628b2a9d2f9 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 14 Jan 2026 14:23:16 +0800 Subject: [PATCH 9/9] Update comments --- .../lib/Transforms/Vectorize/VPlanTransforms.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index b74a513cc128d..04f6a05da7bde 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -3103,19 +3103,21 @@ static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) { HeaderMask->replaceAllUsesWith(EVLMask); } -/// Converts a tail folded vector loop region to step by @llvm.get.vector.length -/// elements instead of VF elements each iteration. +/// Converts a tail folded vector loop region to step by +/// VPInstruction::ExplicitVectorLength elements instead of VF elements each +/// iteration. /// /// - Add a VPEVLBasedIVPHIRecipe and related recipes to \p Plan and -/// replaces all uses except the canonical IV increment of -/// VPCanonicalIVPHIRecipe with a VPEVLBasedIVPHIRecipe. VPCanonicalIVPHIRecipe -/// is used only for loop iterations counting after this transformation. +/// replaces all uses except the canonical IV increment of +/// VPCanonicalIVPHIRecipe with a VPEVLBasedIVPHIRecipe. +/// VPCanonicalIVPHIRecipe is used only for loop iterations counting after +/// this transformation. /// /// - The header mask is replaced with a header mask based on the EVL. /// /// - Plans with FORs have a new phi added to keep track of the EVL of the -/// previous iteration, and VPFirstOrderRecurrencePHIRecipes are replaced with -/// @llvm.vp.splice. +/// previous iteration, and VPFirstOrderRecurrencePHIRecipes are replaced with +/// @llvm.vp.splice. /// /// The function uses the following definitions: /// %StartV is the canonical induction start value.