From f8c40c2a4bf93abe3a29bee4860ea609e5b4bf9a Mon Sep 17 00:00:00 2001 From: Pengcheng Wang Date: Thu, 28 May 2026 14:24:35 +0800 Subject: [PATCH 1/2] [RISCV] Custom lower VP_LOAD/VP_STORE of mask vectors For scalable mask vectors (``), `VP_LOAD/VP_STORE` were left at the default `Expand` action. `VectorLegalizer::Expand` falls back to `UnrollVectorOp`, which only supports fixed-length vectors and ends up calling `getVectorNumElements()` on a scalable type, crashing the compiler. This is reachable from `llvm.vp.load` / `llvm.vp.store` of `nxvNi1`. Fix it by: 1. Setting `VP_LOAD`/`VP_STORE` to `Custom` for all legal `BoolVecVTs`. 2. Adding a mask-element-type fast path in `lowerMaskedLoad` / `lowerMaskedStore` that lowers the operation to the mask-register instructions `vlm.v` / `vsm.v` (mirroring the existing path in `lowerFixedLengthVectorLoadToRVV` / `lowerFixedLengthVectorStoreToRVV`). 3. Adds tests in `llvm/test/CodeGen/RISCV/rvv/vpload.ll` and `vpstore.ll` for `nxv{1,2,4,8,16,32,64}i1. This fixes #199896. This PR is done by TRAE (ByteDance's coding agent). --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 30 ++++++++ llvm/test/CodeGen/RISCV/rvv/vpload.ll | 80 +++++++++++++++++++++ llvm/test/CodeGen/RISCV/rvv/vpstore.ll | 80 +++++++++++++++++++++ 3 files changed, 190 insertions(+) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 8add1d540c3ca..a56af20f50926 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -979,6 +979,8 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM, Expand); setOperationAction(ISD::VP_MERGE, VT, Custom); + setOperationAction({ISD::VP_LOAD, ISD::VP_STORE}, VT, Custom); + setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON, ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_POISON}, VT, Custom); @@ -13762,6 +13764,21 @@ SDValue RISCVTargetLowering::lowerMaskedLoad(SDValue Op, MVT XLenVT = Subtarget.getXLenVT(); + // VP_LOAD of a mask vector lowers directly to vlm.v on the mask register. + if (VT.getVectorElementType() == MVT::i1) { + assert(isa(Op) && + "MaskedLoad of i1 vector should not reach here"); + if (!VL) + VL = getDefaultVLOps(VT, VT, DL, DAG, Subtarget).second; + SDValue IntID = DAG.getTargetConstant(Intrinsic::riscv_vlm, DL, XLenVT); + SDValue Ops[] = {Chain, IntID, BasePtr, VL}; + SDVTList VTs = DAG.getVTList({VT, MVT::Other}); + SDValue Result = + DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, + MMO); + return DAG.getMergeValues({Result, Result.getValue(1)}, DL); + } + MVT ContainerVT = VT; if (VT.isFixedLengthVector()) { ContainerVT = getContainerForFixedLengthVector(VT); @@ -13909,6 +13926,19 @@ SDValue RISCVTargetLowering::lowerMaskedStore(SDValue Op, MVT VT = Val.getSimpleValueType(); MVT XLenVT = Subtarget.getXLenVT(); + // VP_STORE of a mask vector lowers directly to vsm.v on the mask register. + if (VT.getVectorElementType() == MVT::i1) { + assert(isa(Op) && + "MaskedStore of i1 vector should not reach here"); + if (!VL) + VL = getDefaultVLOps(VT, VT, DL, DAG, Subtarget).second; + SDValue IntID = DAG.getTargetConstant(Intrinsic::riscv_vsm, DL, XLenVT); + return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, + DAG.getVTList(MVT::Other), + {Chain, IntID, Val, BasePtr, VL}, MemVT, + MMO); + } + MVT ContainerVT = VT; if (VT.isFixedLengthVector()) { ContainerVT = getContainerForFixedLengthVector(VT); diff --git a/llvm/test/CodeGen/RISCV/rvv/vpload.ll b/llvm/test/CodeGen/RISCV/rvv/vpload.ll index 2ece316c7e54a..74a3c45734bfd 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vpload.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vpload.ll @@ -560,3 +560,83 @@ define @vpload_all_active_nxv8i8(ptr %ptr) { %load = call @llvm.vp.load.nxv8i8.p0(ptr %ptr, splat (i1 true), i32 %evl) ret %load } + +define @vpload_nxv1i1(ptr %ptr, i32 zeroext %evl) { +; CHECK-LABEL: vpload_nxv1i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, mf8, ta, ma +; CHECK-NEXT: vlm.v v0, (a0) +; CHECK-NEXT: ret + %load = call @llvm.vp.load.nxv1i1.p0(ptr %ptr, splat (i1 true), i32 %evl) + ret %load +} + +define @vpload_nxv2i1(ptr %ptr, i32 zeroext %evl) { +; CHECK-LABEL: vpload_nxv2i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, mf4, ta, ma +; CHECK-NEXT: vlm.v v0, (a0) +; CHECK-NEXT: ret + %load = call @llvm.vp.load.nxv2i1.p0(ptr %ptr, splat (i1 true), i32 %evl) + ret %load +} + +define @vpload_nxv4i1(ptr %ptr, i32 zeroext %evl) { +; CHECK-LABEL: vpload_nxv4i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, ma +; CHECK-NEXT: vlm.v v0, (a0) +; CHECK-NEXT: ret + %load = call @llvm.vp.load.nxv4i1.p0(ptr %ptr, splat (i1 true), i32 %evl) + ret %load +} + +define @vpload_nxv8i1(ptr %ptr, i32 zeroext %evl) { +; CHECK-LABEL: vpload_nxv8i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, ma +; CHECK-NEXT: vlm.v v0, (a0) +; CHECK-NEXT: ret + %load = call @llvm.vp.load.nxv8i1.p0(ptr %ptr, splat (i1 true), i32 %evl) + ret %load +} + +define @vpload_nxv16i1(ptr %ptr, i32 zeroext %evl) { +; CHECK-LABEL: vpload_nxv16i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma +; CHECK-NEXT: vlm.v v0, (a0) +; CHECK-NEXT: ret + %load = call @llvm.vp.load.nxv16i1.p0(ptr %ptr, splat (i1 true), i32 %evl) + ret %load +} + +define @vpload_nxv32i1(ptr %ptr, i32 zeroext %evl) { +; CHECK-LABEL: vpload_nxv32i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, m4, ta, ma +; CHECK-NEXT: vlm.v v0, (a0) +; CHECK-NEXT: ret + %load = call @llvm.vp.load.nxv32i1.p0(ptr %ptr, splat (i1 true), i32 %evl) + ret %load +} + +define @vpload_nxv64i1(ptr %ptr, i32 zeroext %evl) { +; CHECK-LABEL: vpload_nxv64i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, m8, ta, ma +; CHECK-NEXT: vlm.v v0, (a0) +; CHECK-NEXT: ret + %load = call @llvm.vp.load.nxv64i1.p0(ptr %ptr, splat (i1 true), i32 %evl) + ret %load +} + +define @vpload_nxv16i1_masked(ptr %ptr, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpload_nxv16i1_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma +; CHECK-NEXT: vlm.v v0, (a0) +; CHECK-NEXT: ret + %load = call @llvm.vp.load.nxv16i1.p0(ptr %ptr, %m, i32 %evl) + ret %load +} diff --git a/llvm/test/CodeGen/RISCV/rvv/vpstore.ll b/llvm/test/CodeGen/RISCV/rvv/vpstore.ll index 9fd8b9d23cb5e..8d16c213551ed 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vpstore.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vpstore.ll @@ -462,3 +462,83 @@ define void @vpstore_all_active_nxv8i8( %val, ptr %ptr) { call void @llvm.vp.store.nxv8i8.p0( %val, ptr %ptr, splat (i1 true), i32 %evl) ret void } + +define void @vpstore_nxv1i1( %val, ptr %ptr, i32 zeroext %evl) { +; CHECK-LABEL: vpstore_nxv1i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, mf8, ta, ma +; CHECK-NEXT: vsm.v v0, (a0) +; CHECK-NEXT: ret + call void @llvm.vp.store.nxv1i1.p0( %val, ptr %ptr, splat (i1 true), i32 %evl) + ret void +} + +define void @vpstore_nxv2i1( %val, ptr %ptr, i32 zeroext %evl) { +; CHECK-LABEL: vpstore_nxv2i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, mf4, ta, ma +; CHECK-NEXT: vsm.v v0, (a0) +; CHECK-NEXT: ret + call void @llvm.vp.store.nxv2i1.p0( %val, ptr %ptr, splat (i1 true), i32 %evl) + ret void +} + +define void @vpstore_nxv4i1( %val, ptr %ptr, i32 zeroext %evl) { +; CHECK-LABEL: vpstore_nxv4i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, ma +; CHECK-NEXT: vsm.v v0, (a0) +; CHECK-NEXT: ret + call void @llvm.vp.store.nxv4i1.p0( %val, ptr %ptr, splat (i1 true), i32 %evl) + ret void +} + +define void @vpstore_nxv8i1( %val, ptr %ptr, i32 zeroext %evl) { +; CHECK-LABEL: vpstore_nxv8i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, m1, ta, ma +; CHECK-NEXT: vsm.v v0, (a0) +; CHECK-NEXT: ret + call void @llvm.vp.store.nxv8i1.p0( %val, ptr %ptr, splat (i1 true), i32 %evl) + ret void +} + +define void @vpstore_nxv16i1( %val, ptr %ptr, i32 zeroext %evl) { +; CHECK-LABEL: vpstore_nxv16i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma +; CHECK-NEXT: vsm.v v0, (a0) +; CHECK-NEXT: ret + call void @llvm.vp.store.nxv16i1.p0( %val, ptr %ptr, splat (i1 true), i32 %evl) + ret void +} + +define void @vpstore_nxv32i1( %val, ptr %ptr, i32 zeroext %evl) { +; CHECK-LABEL: vpstore_nxv32i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, m4, ta, ma +; CHECK-NEXT: vsm.v v0, (a0) +; CHECK-NEXT: ret + call void @llvm.vp.store.nxv32i1.p0( %val, ptr %ptr, splat (i1 true), i32 %evl) + ret void +} + +define void @vpstore_nxv64i1( %val, ptr %ptr, i32 zeroext %evl) { +; CHECK-LABEL: vpstore_nxv64i1: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, m8, ta, ma +; CHECK-NEXT: vsm.v v0, (a0) +; CHECK-NEXT: ret + call void @llvm.vp.store.nxv64i1.p0( %val, ptr %ptr, splat (i1 true), i32 %evl) + ret void +} + +define void @vpstore_nxv16i1_masked( %val, ptr %ptr, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpstore_nxv16i1_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma +; CHECK-NEXT: vsm.v v0, (a0) +; CHECK-NEXT: ret + call void @llvm.vp.store.nxv16i1.p0( %val, ptr %ptr, %m, i32 %evl) + ret void +} From bb3b48f139a063a57beb5d6faa818939ebadf1e2 Mon Sep 17 00:00:00 2001 From: Pengcheng Wang Date: Thu, 28 May 2026 17:48:58 +0800 Subject: [PATCH 2/2] clang-format --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index a56af20f50926..820a97ab65700 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -13773,9 +13773,8 @@ SDValue RISCVTargetLowering::lowerMaskedLoad(SDValue Op, SDValue IntID = DAG.getTargetConstant(Intrinsic::riscv_vlm, DL, XLenVT); SDValue Ops[] = {Chain, IntID, BasePtr, VL}; SDVTList VTs = DAG.getVTList({VT, MVT::Other}); - SDValue Result = - DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, - MMO); + SDValue Result = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, + Ops, MemVT, MMO); return DAG.getMergeValues({Result, Result.getValue(1)}, DL); } @@ -13933,10 +13932,9 @@ SDValue RISCVTargetLowering::lowerMaskedStore(SDValue Op, if (!VL) VL = getDefaultVLOps(VT, VT, DL, DAG, Subtarget).second; SDValue IntID = DAG.getTargetConstant(Intrinsic::riscv_vsm, DL, XLenVT); - return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, - DAG.getVTList(MVT::Other), - {Chain, IntID, Val, BasePtr, VL}, MemVT, - MMO); + return DAG.getMemIntrinsicNode( + ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), + {Chain, IntID, Val, BasePtr, VL}, MemVT, MMO); } MVT ContainerVT = VT;