diff --git a/llvm/lib/CodeGen/InterleavedAccessPass.cpp b/llvm/lib/CodeGen/InterleavedAccessPass.cpp index c1b84c109af8e..df162fca18d92 100644 --- a/llvm/lib/CodeGen/InterleavedAccessPass.cpp +++ b/llvm/lib/CodeGen/InterleavedAccessPass.cpp @@ -661,7 +661,9 @@ bool InterleavedAccessImpl::lowerInterleaveIntrinsic( Instruction *StoredBy = dyn_cast(IntII->user_back()); if (!StoredBy) return false; - if (!isa(StoredBy)) + auto *SI = dyn_cast(StoredBy); + auto *II = dyn_cast(StoredBy); + if (!SI && !II) return false; SmallVector InterleaveValues(IntII->args()); @@ -669,20 +671,28 @@ bool InterleavedAccessImpl::lowerInterleaveIntrinsic( assert(Factor && "unexpected interleave intrinsic"); Value *Mask = nullptr; - if (auto *VPStore = dyn_cast(StoredBy)) { - if (VPStore->getIntrinsicID() != Intrinsic::vp_store) + if (II) { + // Check mask operand. Handle both all-true/false and interleaved mask. + Value *WideMask; + switch (II->getIntrinsicID()) { + default: return false; - - Value *WideMask = VPStore->getOperand(2); + case Intrinsic::vp_store: + WideMask = II->getOperand(2); + break; + case Intrinsic::masked_store: + WideMask = II->getOperand(3); + break; + } Mask = getMask(WideMask, Factor, cast(InterleaveValues[0]->getType())); if (!Mask) return false; - LLVM_DEBUG(dbgs() << "IA: Found a vp.store with interleave intrinsic " - << *IntII << " and factor = " << Factor << "\n"); + LLVM_DEBUG(dbgs() << "IA: Found a vp.store or masked.store with interleave" + << " intrinsic " << *IntII << " and factor = " + << Factor << "\n"); } else { - auto *SI = cast(StoredBy); if (!SI->isSimple()) return false; diff --git a/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp b/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp index 7d1f78996d118..25817b6d2707f 100644 --- a/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp +++ b/llvm/lib/Target/RISCV/RISCVInterleavedAccess.cpp @@ -169,6 +169,17 @@ static bool getMemOperands(unsigned Factor, VectorType *VTy, Type *XLenTy, : Constant::getAllOnesValue(XLenTy); return true; } + case Intrinsic::masked_store: { + Ptr = II->getOperand(1); + Alignment = cast(II->getArgOperand(2))->getAlignValue(); + + assert(Mask && "masked.store needs a mask!"); + + VL = isa(VTy) + ? Builder.CreateElementCount(XLenTy, VTy->getElementCount()) + : Constant::getAllOnesValue(XLenTy); + return true; + } } } diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll index af55aaa8fce86..7e7d11e8384dc 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll @@ -303,3 +303,26 @@ define void @vector_interleave_store_factor8( %a, %v, ptr %p ret void } + +define void @masked_store_factor3( %a, %b, %c, ptr %p) { +; CHECK-LABEL: masked_store_factor3: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma +; CHECK-NEXT: vsseg3e32.v v8, (a0) +; CHECK-NEXT: ret + %v = call @llvm.vector.interleave3( %a, %b, %c) + call void @llvm.masked.store( %v, ptr %p, i32 4, splat (i1 true)) + ret void +} + +define void @masked_store_factor3_masked( %a, %b, %c, ptr %p, %m) { +; CHECK-LABEL: masked_store_factor3_masked: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e32, m1, ta, ma +; CHECK-NEXT: vsseg3e32.v v8, (a0), v0.t +; CHECK-NEXT: ret + %interleaved.mask = call @llvm.vector.interleave3( %m, %m, %m) + %v = call @llvm.vector.interleave3( %a, %b, %c) + call void @llvm.masked.store( %v, ptr %p, i32 4, %interleaved.mask) + ret void +}