From a4c9d41869e278a52fc8abd74cdea1c5fdb5c478 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Thu, 2 Apr 2026 20:05:48 +0800 Subject: [PATCH 1/2] Add select x, (or i1 y, z), y transform --- .../Transforms/Vectorize/VPlanTransforms.cpp | 9 +++++++++ .../LoopVectorize/AArch64/sve-select-cmp.ll | 4 ++-- .../RISCV/select-cmp-reduction.ll | 20 +++++++++++-------- .../RISCV/tail-folding-inloop-reduction.ll | 6 ++++-- .../RISCV/tail-folding-reduction.ll | 6 ++++-- .../LoopVectorize/select-cmp-predicated.ll | 14 ++++++------- 6 files changed, 38 insertions(+), 21 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 9944df4778774..9730c23ee09a6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1458,6 +1458,15 @@ static void simplifyRecipe(VPSingleDefRecipe *Def, VPTypeAnalysis &TypeInfo) { return; } + // select x, (i1 y | z), y -> y | (x && z) + if (CanCreateNewRecipe && + match(Def, m_Select(m_VPValue(X), + m_OneUse(m_c_BinaryOr(m_VPValue(Y), m_VPValue(Z))), + m_Deferred(Y))) && + TypeInfo.inferScalarType(Y)->isIntegerTy(1)) + return Def->replaceAllUsesWith( + Builder.createOr(Y, Builder.createLogicalAnd(X, Z))); + if (match(Def, m_c_Add(m_VPValue(A), m_ZeroInt()))) return Def->replaceAllUsesWith(A); diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-select-cmp.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-select-cmp.ll index 1165d88312817..0a94b8bb39c9d 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-select-cmp.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-select-cmp.ll @@ -169,8 +169,8 @@ define i32 @pred_select_const_i32_from_icmp(ptr noalias nocapture readonly %src1 ; CHECK-VF4IC1: [[MASK:%.*]] = icmp sgt [[VEC_LOAD]], splat (i32 35) ; CHECK-VF4IC1: [[MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr align 4 {{%.*}}, [[MASK]], poison) ; CHECK-VF4IC1-NEXT: [[VEC_ICMP:%.*]] = icmp eq [[MASKED_LOAD]], splat (i32 2) -; CHECK-VF4IC1-NEXT: [[VEC_SEL_TMP:%.*]] = or [[VEC_PHI]], [[VEC_ICMP]] -; CHECK-VF4IC1: [[VEC_SEL:%.*]] = select [[MASK]], [[VEC_SEL_TMP]], [[VEC_PHI]] +; CHECK-VF4IC1: [[VEC_SEL_TMP:%.*]] = select [[MASK]], [[VEC_ICMP]], zeroinitializer +; CHECK-VF4IC1-NEXT: [[VEC_SEL:%.*]] = or [[VEC_PHI]], [[VEC_SEL_TMP]] ; CHECK-VF4IC1: middle.block: ; CHECK-VF4IC1-NEXT: [[OR_RDX:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1( [[VEC_SEL]]) ; CHECK-VF4IC1-NEXT: [[FR:%.*]] = freeze i1 [[OR_RDX]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll index d7bab8fda3fe3..6bf782c81cbf6 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll @@ -18,7 +18,8 @@ define i32 @select_icmp(i32 %x, i32 %y, ptr nocapture readonly %c, i64 %n) { ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], splat (i1 true), i32 [[TMP14]]) ; CHECK-NEXT: [[TMP7:%.*]] = icmp sge [[WIDE_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: [[TMP8]] = call @llvm.vp.merge.nxv4i1( [[TMP7]], splat (i1 true), [[VEC_PHI]], i32 [[TMP14]]) +; CHECK-NEXT: [[TMP3:%.*]] = call @llvm.vp.merge.nxv4i1( splat (i1 true), [[TMP7]], zeroinitializer, i32 [[TMP14]]) +; CHECK-NEXT: [[TMP8]] = or [[VEC_PHI]], [[TMP3]] ; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP14]] to i64 ; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]] ; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]] @@ -67,7 +68,8 @@ define i32 @select_fcmp(float %x, i32 %y, ptr nocapture readonly %c, i64 %n) { ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[C]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP6]], splat (i1 true), i32 [[TMP14]]) ; CHECK-NEXT: [[TMP7:%.*]] = fcmp fast uge [[WIDE_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: [[TMP8]] = call @llvm.vp.merge.nxv4i1( [[TMP7]], splat (i1 true), [[VEC_PHI]], i32 [[TMP14]]) +; CHECK-NEXT: [[TMP3:%.*]] = call @llvm.vp.merge.nxv4i1( splat (i1 true), [[TMP7]], zeroinitializer, i32 [[TMP14]]) +; CHECK-NEXT: [[TMP8]] = or [[VEC_PHI]], [[TMP3]] ; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP14]] to i64 ; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]] ; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]] @@ -114,7 +116,8 @@ define i32 @select_const_i32_from_icmp(ptr nocapture readonly %v, i64 %n) { ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], splat (i1 true), i32 [[TMP21]]) ; CHECK-NEXT: [[TMP7:%.*]] = icmp ne [[WIDE_LOAD]], splat (i32 3) -; CHECK-NEXT: [[TMP8]] = call @llvm.vp.merge.nxv4i1( [[TMP7]], splat (i1 true), [[VEC_PHI]], i32 [[TMP21]]) +; CHECK-NEXT: [[TMP3:%.*]] = call @llvm.vp.merge.nxv4i1( splat (i1 true), [[TMP7]], zeroinitializer, i32 [[TMP21]]) +; CHECK-NEXT: [[TMP8]] = or [[VEC_PHI]], [[TMP3]] ; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP21]] to i64 ; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]] ; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]] @@ -161,7 +164,8 @@ define i32 @select_i32_from_icmp(ptr nocapture readonly %v, i32 %a, i32 %b, i64 ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], splat (i1 true), i32 [[TMP21]]) ; CHECK-NEXT: [[TMP7:%.*]] = icmp ne [[WIDE_LOAD]], splat (i32 3) -; CHECK-NEXT: [[TMP8]] = call @llvm.vp.merge.nxv4i1( [[TMP7]], splat (i1 true), [[VEC_PHI]], i32 [[TMP21]]) +; CHECK-NEXT: [[TMP3:%.*]] = call @llvm.vp.merge.nxv4i1( splat (i1 true), [[TMP7]], zeroinitializer, i32 [[TMP21]]) +; CHECK-NEXT: [[TMP8]] = or [[VEC_PHI]], [[TMP3]] ; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP21]] to i64 ; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]] ; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]] @@ -208,7 +212,8 @@ define i32 @select_const_i32_from_fcmp(ptr nocapture readonly %v, i64 %n) { ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP6]], splat (i1 true), i32 [[TMP21]]) ; CHECK-NEXT: [[TMP7:%.*]] = fcmp fast one [[WIDE_LOAD]], splat (float 3.000000e+00) -; CHECK-NEXT: [[TMP8]] = call @llvm.vp.merge.nxv4i1( [[TMP7]], splat (i1 true), [[VEC_PHI]], i32 [[TMP21]]) +; CHECK-NEXT: [[TMP3:%.*]] = call @llvm.vp.merge.nxv4i1( splat (i1 true), [[TMP7]], zeroinitializer, i32 [[TMP21]]) +; CHECK-NEXT: [[TMP8]] = or [[VEC_PHI]], [[TMP3]] ; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP21]] to i64 ; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]] ; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]] @@ -322,9 +327,8 @@ define i32 @pred_select_const_i32_from_icmp(ptr noalias nocapture readonly %src1 ; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[SRC2]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP8]], [[TMP7]], i32 [[TMP17]]) ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq [[WIDE_MASKED_LOAD]], splat (i32 2) -; CHECK-NEXT: [[TMP10:%.*]] = or [[VEC_PHI]], [[TMP9]] -; CHECK-NEXT: [[PREDPHI1:%.*]] = select [[TMP7]], [[TMP10]], [[VEC_PHI]] -; CHECK-NEXT: [[PREDPHI]] = call @llvm.vp.merge.nxv4i1( splat (i1 true), [[PREDPHI1]], [[VEC_PHI]], i32 [[TMP17]]) +; CHECK-NEXT: [[TMP5:%.*]] = select [[TMP7]], [[TMP9]], zeroinitializer +; CHECK-NEXT: [[PREDPHI]] = call @llvm.vp.merge.nxv4i1( [[TMP5]], splat (i1 true), [[VEC_PHI]], i32 [[TMP17]]) ; CHECK-NEXT: [[TMP21:%.*]] = zext i32 [[TMP17]] to i64 ; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP21]], [[INDEX]] ; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP21]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-inloop-reduction.ll index 35008d3eea092..fd8c3912b6e3c 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-inloop-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-inloop-reduction.ll @@ -1487,7 +1487,8 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) { ; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]] ; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], splat (i1 true), i32 [[TMP11]]) ; IF-EVL-NEXT: [[TMP13:%.*]] = icmp slt [[VP_OP_LOAD]], splat (i32 3) -; IF-EVL-NEXT: [[TMP16]] = call @llvm.vp.merge.nxv4i1( [[TMP13]], splat (i1 true), [[VEC_PHI]], i32 [[TMP11]]) +; IF-EVL-NEXT: [[TMP3:%.*]] = call @llvm.vp.merge.nxv4i1( splat (i1 true), [[TMP13]], zeroinitializer, i32 [[TMP11]]) +; IF-EVL-NEXT: [[TMP16]] = or [[VEC_PHI]], [[TMP3]] ; IF-EVL-NEXT: [[TMP9:%.*]] = zext i32 [[TMP11]] to i64 ; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[EVL_BASED_IV]] ; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]] @@ -1579,7 +1580,8 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) { ; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]] ; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP12]], splat (i1 true), i32 [[TMP11]]) ; IF-EVL-NEXT: [[TMP13:%.*]] = fcmp fast olt [[VP_OP_LOAD]], splat (float 3.000000e+00) -; IF-EVL-NEXT: [[TMP16]] = call @llvm.vp.merge.nxv4i1( [[TMP13]], splat (i1 true), [[VEC_PHI]], i32 [[TMP11]]) +; IF-EVL-NEXT: [[TMP3:%.*]] = call @llvm.vp.merge.nxv4i1( splat (i1 true), [[TMP13]], zeroinitializer, i32 [[TMP11]]) +; IF-EVL-NEXT: [[TMP16]] = or [[VEC_PHI]], [[TMP3]] ; IF-EVL-NEXT: [[TMP9:%.*]] = zext i32 [[TMP11]] to i64 ; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[EVL_BASED_IV]] ; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll index 6cb9a10856f05..18f8ad4699d8c 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll @@ -1539,7 +1539,8 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) { ; IF-EVL-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]] ; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP11]], splat (i1 true), i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP13:%.*]] = icmp slt [[VP_OP_LOAD]], splat (i32 3) -; IF-EVL-NEXT: [[TMP15]] = call @llvm.vp.merge.nxv4i1( [[TMP13]], splat (i1 true), [[VEC_PHI]], i32 [[TMP9]]) +; IF-EVL-NEXT: [[TMP3:%.*]] = call @llvm.vp.merge.nxv4i1( splat (i1 true), [[TMP13]], zeroinitializer, i32 [[TMP9]]) +; IF-EVL-NEXT: [[TMP15]] = or [[VEC_PHI]], [[TMP3]] ; IF-EVL-NEXT: [[TMP16:%.*]] = zext i32 [[TMP9]] to i64 ; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]] ; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP16]] @@ -1631,7 +1632,8 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) { ; IF-EVL-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]] ; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP11]], splat (i1 true), i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP13:%.*]] = fcmp fast olt [[VP_OP_LOAD]], splat (float 3.000000e+00) -; IF-EVL-NEXT: [[TMP15]] = call @llvm.vp.merge.nxv4i1( [[TMP13]], splat (i1 true), [[VEC_PHI]], i32 [[TMP9]]) +; IF-EVL-NEXT: [[TMP3:%.*]] = call @llvm.vp.merge.nxv4i1( splat (i1 true), [[TMP13]], zeroinitializer, i32 [[TMP9]]) +; IF-EVL-NEXT: [[TMP15]] = or [[VEC_PHI]], [[TMP3]] ; IF-EVL-NEXT: [[TMP16:%.*]] = zext i32 [[TMP9]] to i64 ; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]] ; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP16]] diff --git a/llvm/test/Transforms/LoopVectorize/select-cmp-predicated.ll b/llvm/test/Transforms/LoopVectorize/select-cmp-predicated.ll index 8ab7ea85ea7c7..6e6eea11647df 100644 --- a/llvm/test/Transforms/LoopVectorize/select-cmp-predicated.ll +++ b/llvm/test/Transforms/LoopVectorize/select-cmp-predicated.ll @@ -39,8 +39,8 @@ define i32 @pred_select_const_i32_from_icmp(ptr noalias nocapture readonly %src1 ; CHECK-VF2IC1: [[PRED_LOAD_CONTINUE2]]: ; CHECK-VF2IC1-NEXT: [[TMP15:%.*]] = phi <2 x i32> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF1]] ] ; CHECK-VF2IC1-NEXT: [[TMP16:%.*]] = icmp eq <2 x i32> [[TMP15]], splat (i32 2) -; CHECK-VF2IC1-NEXT: [[TMP17:%.*]] = or <2 x i1> [[VEC_PHI]], [[TMP16]] -; CHECK-VF2IC1-NEXT: [[PREDPHI]] = select <2 x i1> [[TMP4]], <2 x i1> [[TMP17]], <2 x i1> [[VEC_PHI]] +; CHECK-VF2IC1-NEXT: [[TMP17:%.*]] = select <2 x i1> [[TMP4]], <2 x i1> [[TMP16]], <2 x i1> zeroinitializer +; CHECK-VF2IC1-NEXT: [[PREDPHI]] = or <2 x i1> [[VEC_PHI]], [[TMP17]] ; CHECK-VF2IC1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[TMP0]], 2 ; CHECK-VF2IC1-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-VF2IC1-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -88,7 +88,7 @@ define i32 @pred_select_const_i32_from_icmp(ptr noalias nocapture readonly %src1 ; CHECK-VF1IC2: [[VECTOR_BODY]]: ; CHECK-VF1IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE3:.*]] ] ; CHECK-VF1IC2-NEXT: [[VEC_PHI:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[PRED_LOAD_CONTINUE3]] ] -; CHECK-VF1IC2-NEXT: [[VEC_PHI2:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[PREDPHI5:%.*]], %[[PRED_LOAD_CONTINUE3]] ] +; CHECK-VF1IC2-NEXT: [[VEC_PHI1:%.*]] = phi i1 [ false, %[[VECTOR_PH]] ], [ [[PREDPHI5:%.*]], %[[PRED_LOAD_CONTINUE3]] ] ; CHECK-VF1IC2-NEXT: [[TMP17:%.*]] = add i64 [[INDEX]], 1 ; CHECK-VF1IC2-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC1]], i64 [[INDEX]] ; CHECK-VF1IC2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[SRC1]], i64 [[TMP17]] @@ -112,10 +112,10 @@ define i32 @pred_select_const_i32_from_icmp(ptr noalias nocapture readonly %src1 ; CHECK-VF1IC2-NEXT: [[TMP11:%.*]] = phi i32 [ poison, %[[PRED_LOAD_CONTINUE]] ], [ [[TMP10]], %[[PRED_LOAD_IF2]] ] ; CHECK-VF1IC2-NEXT: [[TMP12:%.*]] = icmp eq i32 [[TMP8]], 2 ; CHECK-VF1IC2-NEXT: [[TMP13:%.*]] = icmp eq i32 [[TMP11]], 2 -; CHECK-VF1IC2-NEXT: [[TMP14:%.*]] = or i1 [[VEC_PHI]], [[TMP12]] -; CHECK-VF1IC2-NEXT: [[TMP15:%.*]] = or i1 [[VEC_PHI2]], [[TMP13]] -; CHECK-VF1IC2-NEXT: [[PREDPHI]] = select i1 [[TMP4]], i1 [[TMP14]], i1 [[VEC_PHI]] -; CHECK-VF1IC2-NEXT: [[PREDPHI5]] = select i1 [[TMP5]], i1 [[TMP15]], i1 [[VEC_PHI2]] +; CHECK-VF1IC2-NEXT: [[TMP15:%.*]] = select i1 [[TMP4]], i1 [[TMP12]], i1 false +; CHECK-VF1IC2-NEXT: [[PREDPHI]] = or i1 [[VEC_PHI]], [[TMP15]] +; CHECK-VF1IC2-NEXT: [[TMP21:%.*]] = select i1 [[TMP5]], i1 [[TMP13]], i1 false +; CHECK-VF1IC2-NEXT: [[PREDPHI5]] = or i1 [[VEC_PHI1]], [[TMP21]] ; CHECK-VF1IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 ; CHECK-VF1IC2-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-VF1IC2-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] From 07650654e0f58df98c3c525daedbf6846142bc76 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Thu, 2 Apr 2026 23:07:09 +0800 Subject: [PATCH 2/2] Add lhs | (headermask && rhs) transform --- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 8 ++++++++ .../LoopVectorize/RISCV/select-cmp-reduction.ll | 15 +++++---------- .../RISCV/tail-folding-inloop-reduction.ll | 6 ++---- .../LoopVectorize/RISCV/tail-folding-reduction.ll | 6 ++---- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 9730c23ee09a6..0804c0c27a92d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -3150,6 +3150,14 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask, return new VPInterleaveEVLRecipe(*Interleave, EVL, Mask); VPValue *LHS, *RHS; + // lhs | (headermask && rhs) -> vp.merge rhs, true, lhs, evl + if (match(&CurRecipe, + m_c_BinaryOr(m_VPValue(LHS), + m_LogicalAnd(m_Specific(HeaderMask), m_VPValue(RHS))))) + return new VPWidenIntrinsicRecipe( + Intrinsic::vp_merge, {RHS, Plan->getTrue(), LHS, &EVL}, + TypeInfo.inferScalarType(LHS), {}, {}, DL); + if (match(&CurRecipe, m_Select(m_Specific(HeaderMask), m_VPValue(LHS), m_VPValue(RHS)))) return new VPWidenIntrinsicRecipe( diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll index 6bf782c81cbf6..7ed567a910bcf 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll @@ -18,8 +18,7 @@ define i32 @select_icmp(i32 %x, i32 %y, ptr nocapture readonly %c, i64 %n) { ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], splat (i1 true), i32 [[TMP14]]) ; CHECK-NEXT: [[TMP7:%.*]] = icmp sge [[WIDE_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: [[TMP3:%.*]] = call @llvm.vp.merge.nxv4i1( splat (i1 true), [[TMP7]], zeroinitializer, i32 [[TMP14]]) -; CHECK-NEXT: [[TMP8]] = or [[VEC_PHI]], [[TMP3]] +; CHECK-NEXT: [[TMP8]] = call @llvm.vp.merge.nxv4i1( [[TMP7]], splat (i1 true), [[VEC_PHI]], i32 [[TMP14]]) ; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP14]] to i64 ; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]] ; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]] @@ -68,8 +67,7 @@ define i32 @select_fcmp(float %x, i32 %y, ptr nocapture readonly %c, i64 %n) { ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[C]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP6]], splat (i1 true), i32 [[TMP14]]) ; CHECK-NEXT: [[TMP7:%.*]] = fcmp fast uge [[WIDE_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: [[TMP3:%.*]] = call @llvm.vp.merge.nxv4i1( splat (i1 true), [[TMP7]], zeroinitializer, i32 [[TMP14]]) -; CHECK-NEXT: [[TMP8]] = or [[VEC_PHI]], [[TMP3]] +; CHECK-NEXT: [[TMP8]] = call @llvm.vp.merge.nxv4i1( [[TMP7]], splat (i1 true), [[VEC_PHI]], i32 [[TMP14]]) ; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP14]] to i64 ; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]] ; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]] @@ -116,8 +114,7 @@ define i32 @select_const_i32_from_icmp(ptr nocapture readonly %v, i64 %n) { ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], splat (i1 true), i32 [[TMP21]]) ; CHECK-NEXT: [[TMP7:%.*]] = icmp ne [[WIDE_LOAD]], splat (i32 3) -; CHECK-NEXT: [[TMP3:%.*]] = call @llvm.vp.merge.nxv4i1( splat (i1 true), [[TMP7]], zeroinitializer, i32 [[TMP21]]) -; CHECK-NEXT: [[TMP8]] = or [[VEC_PHI]], [[TMP3]] +; CHECK-NEXT: [[TMP8]] = call @llvm.vp.merge.nxv4i1( [[TMP7]], splat (i1 true), [[VEC_PHI]], i32 [[TMP21]]) ; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP21]] to i64 ; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]] ; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]] @@ -164,8 +161,7 @@ define i32 @select_i32_from_icmp(ptr nocapture readonly %v, i32 %a, i32 %b, i64 ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP6]], splat (i1 true), i32 [[TMP21]]) ; CHECK-NEXT: [[TMP7:%.*]] = icmp ne [[WIDE_LOAD]], splat (i32 3) -; CHECK-NEXT: [[TMP3:%.*]] = call @llvm.vp.merge.nxv4i1( splat (i1 true), [[TMP7]], zeroinitializer, i32 [[TMP21]]) -; CHECK-NEXT: [[TMP8]] = or [[VEC_PHI]], [[TMP3]] +; CHECK-NEXT: [[TMP8]] = call @llvm.vp.merge.nxv4i1( [[TMP7]], splat (i1 true), [[VEC_PHI]], i32 [[TMP21]]) ; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP21]] to i64 ; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]] ; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]] @@ -212,8 +208,7 @@ define i32 @select_const_i32_from_fcmp(ptr nocapture readonly %v, i64 %n) { ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[V]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = call @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP6]], splat (i1 true), i32 [[TMP21]]) ; CHECK-NEXT: [[TMP7:%.*]] = fcmp fast one [[WIDE_LOAD]], splat (float 3.000000e+00) -; CHECK-NEXT: [[TMP3:%.*]] = call @llvm.vp.merge.nxv4i1( splat (i1 true), [[TMP7]], zeroinitializer, i32 [[TMP21]]) -; CHECK-NEXT: [[TMP8]] = or [[VEC_PHI]], [[TMP3]] +; CHECK-NEXT: [[TMP8]] = call @llvm.vp.merge.nxv4i1( [[TMP7]], splat (i1 true), [[VEC_PHI]], i32 [[TMP21]]) ; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP21]] to i64 ; CHECK-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[INDEX]] ; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-inloop-reduction.ll index fd8c3912b6e3c..35008d3eea092 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-inloop-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-inloop-reduction.ll @@ -1487,8 +1487,7 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) { ; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]] ; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP12]], splat (i1 true), i32 [[TMP11]]) ; IF-EVL-NEXT: [[TMP13:%.*]] = icmp slt [[VP_OP_LOAD]], splat (i32 3) -; IF-EVL-NEXT: [[TMP3:%.*]] = call @llvm.vp.merge.nxv4i1( splat (i1 true), [[TMP13]], zeroinitializer, i32 [[TMP11]]) -; IF-EVL-NEXT: [[TMP16]] = or [[VEC_PHI]], [[TMP3]] +; IF-EVL-NEXT: [[TMP16]] = call @llvm.vp.merge.nxv4i1( [[TMP13]], splat (i1 true), [[VEC_PHI]], i32 [[TMP11]]) ; IF-EVL-NEXT: [[TMP9:%.*]] = zext i32 [[TMP11]] to i64 ; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[EVL_BASED_IV]] ; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]] @@ -1580,8 +1579,7 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) { ; IF-EVL-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]] ; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP12]], splat (i1 true), i32 [[TMP11]]) ; IF-EVL-NEXT: [[TMP13:%.*]] = fcmp fast olt [[VP_OP_LOAD]], splat (float 3.000000e+00) -; IF-EVL-NEXT: [[TMP3:%.*]] = call @llvm.vp.merge.nxv4i1( splat (i1 true), [[TMP13]], zeroinitializer, i32 [[TMP11]]) -; IF-EVL-NEXT: [[TMP16]] = or [[VEC_PHI]], [[TMP3]] +; IF-EVL-NEXT: [[TMP16]] = call @llvm.vp.merge.nxv4i1( [[TMP13]], splat (i1 true), [[VEC_PHI]], i32 [[TMP11]]) ; IF-EVL-NEXT: [[TMP9:%.*]] = zext i32 [[TMP11]] to i64 ; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP9]], [[EVL_BASED_IV]] ; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP9]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll index 18f8ad4699d8c..6cb9a10856f05 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-reduction.ll @@ -1539,8 +1539,7 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) { ; IF-EVL-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]] ; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4i32.p0(ptr align 4 [[TMP11]], splat (i1 true), i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP13:%.*]] = icmp slt [[VP_OP_LOAD]], splat (i32 3) -; IF-EVL-NEXT: [[TMP3:%.*]] = call @llvm.vp.merge.nxv4i1( splat (i1 true), [[TMP13]], zeroinitializer, i32 [[TMP9]]) -; IF-EVL-NEXT: [[TMP15]] = or [[VEC_PHI]], [[TMP3]] +; IF-EVL-NEXT: [[TMP15]] = call @llvm.vp.merge.nxv4i1( [[TMP13]], splat (i1 true), [[VEC_PHI]], i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP16:%.*]] = zext i32 [[TMP9]] to i64 ; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]] ; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP16]] @@ -1632,8 +1631,7 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) { ; IF-EVL-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[EVL_BASED_IV]] ; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call @llvm.vp.load.nxv4f32.p0(ptr align 4 [[TMP11]], splat (i1 true), i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP13:%.*]] = fcmp fast olt [[VP_OP_LOAD]], splat (float 3.000000e+00) -; IF-EVL-NEXT: [[TMP3:%.*]] = call @llvm.vp.merge.nxv4i1( splat (i1 true), [[TMP13]], zeroinitializer, i32 [[TMP9]]) -; IF-EVL-NEXT: [[TMP15]] = or [[VEC_PHI]], [[TMP3]] +; IF-EVL-NEXT: [[TMP15]] = call @llvm.vp.merge.nxv4i1( [[TMP13]], splat (i1 true), [[VEC_PHI]], i32 [[TMP9]]) ; IF-EVL-NEXT: [[TMP16:%.*]] = zext i32 [[TMP9]] to i64 ; IF-EVL-NEXT: [[INDEX_EVL_NEXT]] = add i64 [[TMP16]], [[EVL_BASED_IV]] ; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP16]]