From 5d53907ec9f392ff2204c1141fe84352807623ba Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 28 May 2026 09:41:55 +0000 Subject: [PATCH 1/5] [RISCV] Add VP_LOAD/VP_STORE support for mask (i1) vector types This fixes a crash where the vector legalizer tried to unroll a scalable vector vp_load of a mask type (e.g. nxv16i1), which called getVectorNumElements() on a scalable vector. The loop vectorizer can produce vp.load/vp.store of mask types when loading _Bool arrays that are subsequently used as booleans. The RISC-V backend was missing Custom lowering for VP_LOAD/VP_STORE on BoolVecVTs, causing the vector legalizer to fall through to Expand/Unroll which doesn't support scalable vectors. The fix adds VP_LOAD/VP_STORE as Custom for all boolean vector types and handles them in lowerMaskedLoad/lowerMaskedStore by emitting riscv_vlm/riscv_vsm intrinsics respectively. Fixes llvm/llvm-project#199896 Co-authored-by: lukel97 <2488460+lukel97@users.noreply.github.com> --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 27 ++++++++ llvm/test/CodeGen/RISCV/rvv/vpload-mask.ll | 73 +++++++++++++++++++++ llvm/test/CodeGen/RISCV/rvv/vpstore-mask.ll | 73 +++++++++++++++++++++ 3 files changed, 173 insertions(+) create mode 100644 llvm/test/CodeGen/RISCV/rvv/vpload-mask.ll create mode 100644 llvm/test/CodeGen/RISCV/rvv/vpstore-mask.ll diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 1148b801530a0..05fc0e4b3a40c 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -982,6 +982,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom); setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom); + setOperationAction({ISD::VP_LOAD, ISD::VP_STORE}, VT, Custom); + setOperationPromotedToType( {ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT}, VT, MVT::getVectorVT(MVT::i8, VT.getVectorElementCount())); @@ -13455,6 +13457,22 @@ SDValue RISCVTargetLowering::lowerMaskedLoad(SDValue Op, if (!VL) VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; + // Mask vector types (i1 element) use vlm for loading. + if (VT.getVectorElementType() == MVT::i1) { + SDValue IntID = + DAG.getTargetConstant(Intrinsic::riscv_vlm, DL, XLenVT); + SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other}); + SmallVector Ops{Chain, IntID, BasePtr, VL}; + SDValue Result = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, + Ops, MemVT, MMO); + Chain = Result.getValue(1); + + if (VT.isFixedLengthVector()) + Result = convertFromScalableVector(VT, Result, DAG, Subtarget); + + return DAG.getMergeValues({Result, Chain}, DL); + } + SDValue ExpandingVL; if (!IsUnmasked && IsExpandingLoad) { ExpandingVL = VL; @@ -13603,6 +13621,15 @@ SDValue RISCVTargetLowering::lowerMaskedStore(SDValue Op, if (!VL) VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; + // Mask vector types (i1 element) use vsm for storing. + if (VT.getVectorElementType() == MVT::i1) { + SDValue IntID = + DAG.getTargetConstant(Intrinsic::riscv_vsm, DL, XLenVT); + SmallVector Ops{Chain, IntID, Val, BasePtr, VL}; + return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, + DAG.getVTList(MVT::Other), Ops, MemVT, MMO); + } + if (IsCompressingStore) { Val = DAG.getNode( ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT, diff --git a/llvm/test/CodeGen/RISCV/rvv/vpload-mask.ll b/llvm/test/CodeGen/RISCV/rvv/vpload-mask.ll new file mode 100644 index 0000000000000..7d4e1e6856f6b --- /dev/null +++ b/llvm/test/CodeGen/RISCV/rvv/vpload-mask.ll @@ -0,0 +1,73 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s + +define @vpload_nxv1i1(ptr %ptr, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpload_nxv1i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, mf8, ta, ma +; CHECK-NEXT: vlm.v v0, (a0) +; CHECK-NEXT: ret + %load = call @llvm.vp.load.nxv1i1.p0(ptr %ptr, %m, i32 %evl) + ret %load +} + +define @vpload_nxv2i1(ptr %ptr, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpload_nxv2i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, mf4, ta, ma +; CHECK-NEXT: vlm.v v0, (a0) +; CHECK-NEXT: ret + %load = call @llvm.vp.load.nxv2i1.p0(ptr %ptr, %m, i32 %evl) + ret %load +} + +define @vpload_nxv4i1(ptr %ptr, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpload_nxv4i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, ma +; CHECK-NEXT: vlm.v v0, (a0) +; CHECK-NEXT: ret + %load = call @llvm.vp.load.nxv4i1.p0(ptr %ptr, %m, i32 %evl) + ret %load +} + +define @vpload_nxv8i1(ptr %ptr, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpload_nxv8i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, ma +; CHECK-NEXT: vlm.v v0, (a0) +; CHECK-NEXT: ret + %load = call @llvm.vp.load.nxv8i1.p0(ptr %ptr, %m, i32 %evl) + ret %load +} + +define @vpload_nxv16i1(ptr %ptr, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpload_nxv16i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma +; CHECK-NEXT: vlm.v v0, (a0) +; CHECK-NEXT: ret + %load = call @llvm.vp.load.nxv16i1.p0(ptr %ptr, %m, i32 %evl) + ret %load +} + +define @vpload_nxv32i1(ptr %ptr, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpload_nxv32i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, m4, ta, ma +; CHECK-NEXT: vlm.v v0, (a0) +; CHECK-NEXT: ret + %load = call @llvm.vp.load.nxv32i1.p0(ptr %ptr, %m, i32 %evl) + ret %load +} + +define @vpload_nxv64i1(ptr %ptr, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpload_nxv64i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, m8, ta, ma +; CHECK-NEXT: vlm.v v0, (a0) +; CHECK-NEXT: ret + %load = call @llvm.vp.load.nxv64i1.p0(ptr %ptr, %m, i32 %evl) + ret %load +} diff --git a/llvm/test/CodeGen/RISCV/rvv/vpstore-mask.ll b/llvm/test/CodeGen/RISCV/rvv/vpstore-mask.ll new file mode 100644 index 0000000000000..a3dbedc35d38d --- /dev/null +++ b/llvm/test/CodeGen/RISCV/rvv/vpstore-mask.ll @@ -0,0 +1,73 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck %s +; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s + +define void @vpstore_nxv1i1( %val, ptr %ptr, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpstore_nxv1i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, mf8, ta, ma +; CHECK-NEXT: vsm.v v0, (a0) +; CHECK-NEXT: ret + call void @llvm.vp.store.nxv1i1.p0( %val, ptr %ptr, %m, i32 %evl) + ret void +} + +define void @vpstore_nxv2i1( %val, ptr %ptr, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpstore_nxv2i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, mf4, ta, ma +; CHECK-NEXT: vsm.v v0, (a0) +; CHECK-NEXT: ret + call void @llvm.vp.store.nxv2i1.p0( %val, ptr %ptr, %m, i32 %evl) + ret void +} + +define void @vpstore_nxv4i1( %val, ptr %ptr, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpstore_nxv4i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, ma +; CHECK-NEXT: vsm.v v0, (a0) +; CHECK-NEXT: ret + call void @llvm.vp.store.nxv4i1.p0( %val, ptr %ptr, %m, i32 %evl) + ret void +} + +define void @vpstore_nxv8i1( %val, ptr %ptr, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpstore_nxv8i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, ma +; CHECK-NEXT: vsm.v v0, (a0) +; CHECK-NEXT: ret + call void @llvm.vp.store.nxv8i1.p0( %val, ptr %ptr, %m, i32 %evl) + ret void +} + +define void @vpstore_nxv16i1( %val, ptr %ptr, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpstore_nxv16i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma +; CHECK-NEXT: vsm.v v0, (a0) +; CHECK-NEXT: ret + call void @llvm.vp.store.nxv16i1.p0( %val, ptr %ptr, %m, i32 %evl) + ret void +} + +define void @vpstore_nxv32i1( %val, ptr %ptr, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpstore_nxv32i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, m4, ta, ma +; CHECK-NEXT: vsm.v v0, (a0) +; CHECK-NEXT: ret + call void @llvm.vp.store.nxv32i1.p0( %val, ptr %ptr, %m, i32 %evl) + ret void +} + +define void @vpstore_nxv64i1( %val, ptr %ptr, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpstore_nxv64i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, m8, ta, ma +; CHECK-NEXT: vsm.v v0, (a0) +; CHECK-NEXT: ret + call void @llvm.vp.store.nxv64i1.p0( %val, ptr %ptr, %m, i32 %evl) + ret void +} From 71f1279b264ef740557913098b6e66f85edba0e6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 28 May 2026 10:06:45 +0000 Subject: [PATCH 2/5] Prevent vp.load/vp.store of i1 types in optimizeMaskToEVL RVV's vlm.v/vsm.m instructions don't support a mask operand, so vp.load/vp.store of i1 types cannot be correctly lowered. The mask would be silently dropped during lowering. Add checks in optimizeMaskToEVL to skip the conversion of masked loads/stores to EVL form when the element type is i1. Co-authored-by: lukel97 <2488460+lukel97@users.noreply.github.com> --- .../Transforms/Vectorize/VPlanTransforms.cpp | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index d1ff014907dc9..0c3441104bfc6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -3088,9 +3088,15 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask, if (match(&CurRecipe, m_MaskedLoad(m_VPValue(Addr), m_RemoveMask(HeaderMask, Mask))) && - !cast(CurRecipe).isReverse()) + !cast(CurRecipe).isReverse()) { + // Skip i1 element types since vp.load of i1 cannot be correctly lowered: + // vlm.v has no mask operand to represent the EVL mask. + if (getLoadStoreType(&cast(CurRecipe).getIngredient()) + ->isIntegerTy(1)) + return nullptr; return new VPWidenLoadEVLRecipe(cast(CurRecipe), Addr, EVL, Mask); + } VPValue *ReversedVal; if (match(&CurRecipe, m_Reverse(m_VPValue(ReversedVal))) && @@ -3098,6 +3104,10 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask, m_MaskedLoad(m_VPValue(EndPtr), m_RemoveMask(HeaderMask, Mask))) && match(EndPtr, m_VecEndPtr(m_VPValue(Addr), m_Specific(&Plan->getVF()))) && cast(ReversedVal)->isReverse()) { + // Skip i1 element types since vp.load of i1 cannot be correctly lowered. + if (getLoadStoreType(&cast(ReversedVal)->getIngredient()) + ->isIntegerTy(1)) + return nullptr; auto *LoadR = new VPWidenLoadEVLRecipe( *cast(ReversedVal), AdjustEndPtr(EndPtr), EVL, Mask); LoadR->insertBefore(&CurRecipe); @@ -3109,15 +3119,25 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask, VPValue *StoredVal; if (match(&CurRecipe, m_MaskedStore(m_VPValue(Addr), m_VPValue(StoredVal), m_RemoveMask(HeaderMask, Mask))) && - !cast(CurRecipe).isReverse()) + !cast(CurRecipe).isReverse()) { + // Skip i1 element types since vp.store of i1 cannot be correctly lowered: + // vsm.m has no mask operand to represent the EVL mask. + if (getLoadStoreType(&cast(CurRecipe).getIngredient()) + ->isIntegerTy(1)) + return nullptr; return new VPWidenStoreEVLRecipe(cast(CurRecipe), Addr, StoredVal, EVL, Mask); + } if (match(&CurRecipe, m_MaskedStore(m_VPValue(EndPtr), m_Reverse(m_VPValue(ReversedVal)), m_RemoveMask(HeaderMask, Mask))) && match(EndPtr, m_VecEndPtr(m_VPValue(Addr), m_Specific(&Plan->getVF()))) && cast(CurRecipe).isReverse()) { + // Skip i1 element types since vp.store of i1 cannot be correctly lowered. + if (getLoadStoreType(&cast(CurRecipe).getIngredient()) + ->isIntegerTy(1)) + return nullptr; auto *NewReverse = new VPWidenIntrinsicRecipe( Intrinsic::experimental_vp_reverse, {ReversedVal, Plan->getTrue(), &EVL}, From d3666c3f477feeb7cb151a65b418604ebbd43b96 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 28 May 2026 12:29:46 +0000 Subject: [PATCH 3/5] Remove vp.load/vp.store RISCV lowering changes; add crash-fix test The VPlanTransforms.cpp fix (preventing i1 types in optimizeMaskToEVL) is sufficient to fix the crash. Since optimizeEVLMasks runs before the plan cost is computed, i1 loads/stores will remain as masked loads/stores rather than being converted to vp.load/vp.store, avoiding the RISC-V backend crash. Remove the vp.load/vp.store RISCV lowering changes and their CodeGen tests, and add a LoopVectorize test for issue #199896. Co-authored-by: lukel97 <2488460+lukel97@users.noreply.github.com> --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 27 ------- llvm/test/CodeGen/RISCV/rvv/vpload-mask.ll | 73 ------------------- llvm/test/CodeGen/RISCV/rvv/vpstore-mask.ll | 73 ------------------- .../LoopVectorize/RISCV/pr199896.ll | 27 +++++++ 4 files changed, 27 insertions(+), 173 deletions(-) delete mode 100644 llvm/test/CodeGen/RISCV/rvv/vpload-mask.ll delete mode 100644 llvm/test/CodeGen/RISCV/rvv/vpstore-mask.ll create mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/pr199896.ll diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 05fc0e4b3a40c..1148b801530a0 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -982,8 +982,6 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom); setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom); - setOperationAction({ISD::VP_LOAD, ISD::VP_STORE}, VT, Custom); - setOperationPromotedToType( {ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT}, VT, MVT::getVectorVT(MVT::i8, VT.getVectorElementCount())); @@ -13457,22 +13455,6 @@ SDValue RISCVTargetLowering::lowerMaskedLoad(SDValue Op, if (!VL) VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; - // Mask vector types (i1 element) use vlm for loading. - if (VT.getVectorElementType() == MVT::i1) { - SDValue IntID = - DAG.getTargetConstant(Intrinsic::riscv_vlm, DL, XLenVT); - SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other}); - SmallVector Ops{Chain, IntID, BasePtr, VL}; - SDValue Result = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, - Ops, MemVT, MMO); - Chain = Result.getValue(1); - - if (VT.isFixedLengthVector()) - Result = convertFromScalableVector(VT, Result, DAG, Subtarget); - - return DAG.getMergeValues({Result, Chain}, DL); - } - SDValue ExpandingVL; if (!IsUnmasked && IsExpandingLoad) { ExpandingVL = VL; @@ -13621,15 +13603,6 @@ SDValue RISCVTargetLowering::lowerMaskedStore(SDValue Op, if (!VL) VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second; - // Mask vector types (i1 element) use vsm for storing. - if (VT.getVectorElementType() == MVT::i1) { - SDValue IntID = - DAG.getTargetConstant(Intrinsic::riscv_vsm, DL, XLenVT); - SmallVector Ops{Chain, IntID, Val, BasePtr, VL}; - return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, - DAG.getVTList(MVT::Other), Ops, MemVT, MMO); - } - if (IsCompressingStore) { Val = DAG.getNode( ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT, diff --git a/llvm/test/CodeGen/RISCV/rvv/vpload-mask.ll b/llvm/test/CodeGen/RISCV/rvv/vpload-mask.ll deleted file mode 100644 index 7d4e1e6856f6b..0000000000000 --- a/llvm/test/CodeGen/RISCV/rvv/vpload-mask.ll +++ /dev/null @@ -1,73 +0,0 @@ -; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s - -define @vpload_nxv1i1(ptr %ptr, %m, i32 zeroext %evl) { -; CHECK-LABEL: vpload_nxv1i1: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a1, e8, mf8, ta, ma -; CHECK-NEXT: vlm.v v0, (a0) -; CHECK-NEXT: ret - %load = call @llvm.vp.load.nxv1i1.p0(ptr %ptr, %m, i32 %evl) - ret %load -} - -define @vpload_nxv2i1(ptr %ptr, %m, i32 zeroext %evl) { -; CHECK-LABEL: vpload_nxv2i1: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a1, e8, mf4, ta, ma -; CHECK-NEXT: vlm.v v0, (a0) -; CHECK-NEXT: ret - %load = call @llvm.vp.load.nxv2i1.p0(ptr %ptr, %m, i32 %evl) - ret %load -} - -define @vpload_nxv4i1(ptr %ptr, %m, i32 zeroext %evl) { -; CHECK-LABEL: vpload_nxv4i1: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, ma -; CHECK-NEXT: vlm.v v0, (a0) -; CHECK-NEXT: ret - %load = call @llvm.vp.load.nxv4i1.p0(ptr %ptr, %m, i32 %evl) - ret %load -} - -define @vpload_nxv8i1(ptr %ptr, %m, i32 zeroext %evl) { -; CHECK-LABEL: vpload_nxv8i1: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, ma -; CHECK-NEXT: vlm.v v0, (a0) -; CHECK-NEXT: ret - %load = call @llvm.vp.load.nxv8i1.p0(ptr %ptr, %m, i32 %evl) - ret %load -} - -define @vpload_nxv16i1(ptr %ptr, %m, i32 zeroext %evl) { -; CHECK-LABEL: vpload_nxv16i1: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma -; CHECK-NEXT: vlm.v v0, (a0) -; CHECK-NEXT: ret - %load = call @llvm.vp.load.nxv16i1.p0(ptr %ptr, %m, i32 %evl) - ret %load -} - -define @vpload_nxv32i1(ptr %ptr, %m, i32 zeroext %evl) { -; CHECK-LABEL: vpload_nxv32i1: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a1, e8, m4, ta, ma -; CHECK-NEXT: vlm.v v0, (a0) -; CHECK-NEXT: ret - %load = call @llvm.vp.load.nxv32i1.p0(ptr %ptr, %m, i32 %evl) - ret %load -} - -define @vpload_nxv64i1(ptr %ptr, %m, i32 zeroext %evl) { -; CHECK-LABEL: vpload_nxv64i1: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a1, e8, m8, ta, ma -; CHECK-NEXT: vlm.v v0, (a0) -; CHECK-NEXT: ret - %load = call @llvm.vp.load.nxv64i1.p0(ptr %ptr, %m, i32 %evl) - ret %load -} diff --git a/llvm/test/CodeGen/RISCV/rvv/vpstore-mask.ll b/llvm/test/CodeGen/RISCV/rvv/vpstore-mask.ll deleted file mode 100644 index a3dbedc35d38d..0000000000000 --- a/llvm/test/CodeGen/RISCV/rvv/vpstore-mask.ll +++ /dev/null @@ -1,73 +0,0 @@ -; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=riscv32 -mattr=+v -verify-machineinstrs < %s | FileCheck %s -; RUN: llc -mtriple=riscv64 -mattr=+v -verify-machineinstrs < %s | FileCheck %s - -define void @vpstore_nxv1i1( %val, ptr %ptr, %m, i32 zeroext %evl) { -; CHECK-LABEL: vpstore_nxv1i1: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a1, e8, mf8, ta, ma -; CHECK-NEXT: vsm.v v0, (a0) -; CHECK-NEXT: ret - call void @llvm.vp.store.nxv1i1.p0( %val, ptr %ptr, %m, i32 %evl) - ret void -} - -define void @vpstore_nxv2i1( %val, ptr %ptr, %m, i32 zeroext %evl) { -; CHECK-LABEL: vpstore_nxv2i1: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a1, e8, mf4, ta, ma -; CHECK-NEXT: vsm.v v0, (a0) -; CHECK-NEXT: ret - call void @llvm.vp.store.nxv2i1.p0( %val, ptr %ptr, %m, i32 %evl) - ret void -} - -define void @vpstore_nxv4i1( %val, ptr %ptr, %m, i32 zeroext %evl) { -; CHECK-LABEL: vpstore_nxv4i1: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, ma -; CHECK-NEXT: vsm.v v0, (a0) -; CHECK-NEXT: ret - call void @llvm.vp.store.nxv4i1.p0( %val, ptr %ptr, %m, i32 %evl) - ret void -} - -define void @vpstore_nxv8i1( %val, ptr %ptr, %m, i32 zeroext %evl) { -; CHECK-LABEL: vpstore_nxv8i1: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, ma -; CHECK-NEXT: vsm.v v0, (a0) -; CHECK-NEXT: ret - call void @llvm.vp.store.nxv8i1.p0( %val, ptr %ptr, %m, i32 %evl) - ret void -} - -define void @vpstore_nxv16i1( %val, ptr %ptr, %m, i32 zeroext %evl) { -; CHECK-LABEL: vpstore_nxv16i1: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma -; CHECK-NEXT: vsm.v v0, (a0) -; CHECK-NEXT: ret - call void @llvm.vp.store.nxv16i1.p0( %val, ptr %ptr, %m, i32 %evl) - ret void -} - -define void @vpstore_nxv32i1( %val, ptr %ptr, %m, i32 zeroext %evl) { -; CHECK-LABEL: vpstore_nxv32i1: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a1, e8, m4, ta, ma -; CHECK-NEXT: vsm.v v0, (a0) -; CHECK-NEXT: ret - call void @llvm.vp.store.nxv32i1.p0( %val, ptr %ptr, %m, i32 %evl) - ret void -} - -define void @vpstore_nxv64i1( %val, ptr %ptr, %m, i32 zeroext %evl) { -; CHECK-LABEL: vpstore_nxv64i1: -; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a1, e8, m8, ta, ma -; CHECK-NEXT: vsm.v v0, (a0) -; CHECK-NEXT: ret - call void @llvm.vp.store.nxv64i1.p0( %val, ptr %ptr, %m, i32 %evl) - ret void -} diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pr199896.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pr199896.ll new file mode 100644 index 0000000000000..3f3dc662e2bc4 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/RISCV/pr199896.ll @@ -0,0 +1,27 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; RUN: opt -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -mtriple=riscv64 -mattr=+v -S < %s | FileCheck %s + +; Test for https://github.com/llvm/llvm-project/issues/199896 +; This used to crash because optimizeMaskToEVL would create vp.load/vp.store +; of i1 types, which the RISC-V backend cannot lower (vlm.v/vsm.v have no +; mask operand to represent the EVL mask). The fix is to skip i1 element types +; in optimizeMaskToEVL so they remain as masked loads/stores. + +; CHECK-LABEL: define void @copy_bool( +define void @copy_bool(ptr noalias %dst, ptr noalias %src, i64 %n) { +entry: + br label %for.body + +for.body: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] + %src.gep = getelementptr i1, ptr %src, i64 %iv + %val = load i1, ptr %src.gep, align 1 + %dst.gep = getelementptr i1, ptr %dst, i64 %iv + store i1 %val, ptr %dst.gep, align 1 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond = icmp eq i64 %iv.next, %n + br i1 %exitcond, label %for.end, label %for.body + +for.end: + ret void +} From c0ce65ed1fb226bc975aba056afb5f302ff5945d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 28 May 2026 13:14:16 +0000 Subject: [PATCH 4/5] [RISCV] Reject i1 vp.load/store in cost model Revert the LoopVectorize-side guard in optimizeMaskToEVL and model i1 vp.load/vp.store as invalid in RISC-V TTI instead. RISC-V has no native masked memory op with vp.load/vp.store semantics for i1 element types (vlm/vsm lack a mask operand), so plans that rely on these operations should be rejected by profitability/cost checks. This updates both mem intrinsic and intrinsic cost hooks for vp_load/ vp_store i1 vectors and adds a cost-model regression test. Also remove the temporary LoopVectorize regression test added for the previous approach. Co-authored-by: lukel97 <2488460+lukel97@users.noreply.github.com> --- .../Target/RISCV/RISCVTargetTransformInfo.cpp | 14 ++++++++++ .../Transforms/Vectorize/VPlanTransforms.cpp | 24 ++--------------- .../Analysis/CostModel/RISCV/vp-intrinsics.ll | 11 ++++++++ .../LoopVectorize/RISCV/pr199896.ll | 27 ------------------- 4 files changed, 27 insertions(+), 49 deletions(-) delete mode 100644 llvm/test/Transforms/LoopVectorize/RISCV/pr199896.ll diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index fd762f35124b5..14dfc61c8779b 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -1072,6 +1072,12 @@ RISCVTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA, return getGatherScatterOpCost(MICA, CostKind); case Intrinsic::vp_load: case Intrinsic::vp_store: + // RISC-V has no native masked memory instructions matching vp.load/store + // semantics for i1 elements (vlm/vsm don't have a mask operand). Model + // these as invalid so profitability checks reject plans that rely on them. + if (DataTy->isIntOrIntVectorTy(1)) + return InstructionCost::getInvalid(); + [[fallthrough]]; case Intrinsic::masked_load: case Intrinsic::masked_store: return getMaskedMemoryOpCost(MICA, CostKind); @@ -1380,6 +1386,14 @@ RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, TTI::TargetCostKind CostKind) const { auto *RetTy = ICA.getReturnType(); switch (ICA.getID()) { + case Intrinsic::vp_load: + case Intrinsic::vp_store: { + Type *DataTy = ICA.getID() == Intrinsic::vp_store ? ICA.getArgTypes()[0] + : ICA.getReturnType(); + if (DataTy && DataTy->isIntOrIntVectorTy(1)) + return InstructionCost::getInvalid(); + break; + } case Intrinsic::lrint: case Intrinsic::llrint: case Intrinsic::lround: diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 0c3441104bfc6..d1ff014907dc9 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -3088,15 +3088,9 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask, if (match(&CurRecipe, m_MaskedLoad(m_VPValue(Addr), m_RemoveMask(HeaderMask, Mask))) && - !cast(CurRecipe).isReverse()) { - // Skip i1 element types since vp.load of i1 cannot be correctly lowered: - // vlm.v has no mask operand to represent the EVL mask. - if (getLoadStoreType(&cast(CurRecipe).getIngredient()) - ->isIntegerTy(1)) - return nullptr; + !cast(CurRecipe).isReverse()) return new VPWidenLoadEVLRecipe(cast(CurRecipe), Addr, EVL, Mask); - } VPValue *ReversedVal; if (match(&CurRecipe, m_Reverse(m_VPValue(ReversedVal))) && @@ -3104,10 +3098,6 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask, m_MaskedLoad(m_VPValue(EndPtr), m_RemoveMask(HeaderMask, Mask))) && match(EndPtr, m_VecEndPtr(m_VPValue(Addr), m_Specific(&Plan->getVF()))) && cast(ReversedVal)->isReverse()) { - // Skip i1 element types since vp.load of i1 cannot be correctly lowered. - if (getLoadStoreType(&cast(ReversedVal)->getIngredient()) - ->isIntegerTy(1)) - return nullptr; auto *LoadR = new VPWidenLoadEVLRecipe( *cast(ReversedVal), AdjustEndPtr(EndPtr), EVL, Mask); LoadR->insertBefore(&CurRecipe); @@ -3119,25 +3109,15 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask, VPValue *StoredVal; if (match(&CurRecipe, m_MaskedStore(m_VPValue(Addr), m_VPValue(StoredVal), m_RemoveMask(HeaderMask, Mask))) && - !cast(CurRecipe).isReverse()) { - // Skip i1 element types since vp.store of i1 cannot be correctly lowered: - // vsm.m has no mask operand to represent the EVL mask. - if (getLoadStoreType(&cast(CurRecipe).getIngredient()) - ->isIntegerTy(1)) - return nullptr; + !cast(CurRecipe).isReverse()) return new VPWidenStoreEVLRecipe(cast(CurRecipe), Addr, StoredVal, EVL, Mask); - } if (match(&CurRecipe, m_MaskedStore(m_VPValue(EndPtr), m_Reverse(m_VPValue(ReversedVal)), m_RemoveMask(HeaderMask, Mask))) && match(EndPtr, m_VecEndPtr(m_VPValue(Addr), m_Specific(&Plan->getVF()))) && cast(CurRecipe).isReverse()) { - // Skip i1 element types since vp.store of i1 cannot be correctly lowered. - if (getLoadStoreType(&cast(CurRecipe).getIngredient()) - ->isIntegerTy(1)) - return nullptr; auto *NewReverse = new VPWidenIntrinsicRecipe( Intrinsic::experimental_vp_reverse, {ReversedVal, Plan->getTrue(), &EVL}, diff --git a/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll b/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll index 9e8f727978001..6ec81b2cefc73 100644 --- a/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll +++ b/llvm/test/Analysis/CostModel/RISCV/vp-intrinsics.ll @@ -978,6 +978,17 @@ define void @store() { ret void } +define void @i1_vp_memory() { +; CHECK-LABEL: 'i1_vp_memory' +; CHECK-NEXT: Cost Model: Invalid cost for instruction: %l = call @llvm.vp.load.nxv16i1.p0(ptr undef, undef, i32 undef) +; CHECK-NEXT: Cost Model: Invalid cost for instruction: call void @llvm.vp.store.nxv16i1.p0( undef, ptr undef, undef, i32 undef) +; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + %l = call @llvm.vp.load.nxv16i1.p0(ptr undef, undef, i32 undef) + call void @llvm.vp.store.nxv16i1.p0( undef, ptr undef, undef, i32 undef) + ret void +} + define void @gather() { ; ARGBASED-LABEL: 'gather' ; ARGBASED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = call <2 x i8> @llvm.vp.gather.v2i8.v2p0(<2 x ptr> poison, <2 x i1> poison, i32 poison) diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pr199896.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pr199896.ll deleted file mode 100644 index 3f3dc662e2bc4..0000000000000 --- a/llvm/test/Transforms/LoopVectorize/RISCV/pr199896.ll +++ /dev/null @@ -1,27 +0,0 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 -; RUN: opt -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -mtriple=riscv64 -mattr=+v -S < %s | FileCheck %s - -; Test for https://github.com/llvm/llvm-project/issues/199896 -; This used to crash because optimizeMaskToEVL would create vp.load/vp.store -; of i1 types, which the RISC-V backend cannot lower (vlm.v/vsm.v have no -; mask operand to represent the EVL mask). The fix is to skip i1 element types -; in optimizeMaskToEVL so they remain as masked loads/stores. - -; CHECK-LABEL: define void @copy_bool( -define void @copy_bool(ptr noalias %dst, ptr noalias %src, i64 %n) { -entry: - br label %for.body - -for.body: - %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ] - %src.gep = getelementptr i1, ptr %src, i64 %iv - %val = load i1, ptr %src.gep, align 1 - %dst.gep = getelementptr i1, ptr %dst, i64 %iv - store i1 %val, ptr %dst.gep, align 1 - %iv.next = add nuw nsw i64 %iv, 1 - %exitcond = icmp eq i64 %iv.next, %n - br i1 %exitcond, label %for.end, label %for.body - -for.end: - ret void -} From 7c1ef919274a4e4759a1c66d6f2b788d1aeeb823 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 28 May 2026 13:43:45 +0000 Subject: [PATCH 5/5] RISCV: drop redundant i1 vp.load/store mem-cost guard Co-authored-by: lukel97 <2488460+lukel97@users.noreply.github.com> --- llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index 14dfc61c8779b..006d364834524 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -1072,12 +1072,6 @@ RISCVTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA, return getGatherScatterOpCost(MICA, CostKind); case Intrinsic::vp_load: case Intrinsic::vp_store: - // RISC-V has no native masked memory instructions matching vp.load/store - // semantics for i1 elements (vlm/vsm don't have a mask operand). Model - // these as invalid so profitability checks reject plans that rely on them. - if (DataTy->isIntOrIntVectorTy(1)) - return InstructionCost::getInvalid(); - [[fallthrough]]; case Intrinsic::masked_load: case Intrinsic::masked_store: return getMaskedMemoryOpCost(MICA, CostKind);