diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index fd762f35124b5..69f3367e475b2 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -1143,6 +1143,31 @@ InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost( return InstructionCost::getInvalid(); auto *FVTy = cast(VecTy); + // When gaps are only at the tail, for interleaved load, we can emit a wide + // masked load and shufflevectors. For interleaved store, we can emit + // shufflevectors and a wide masked store. The interleaved memory access pass + // will lower them into vlsseg/vssseg intrinsics. + if (UseMaskForGaps) { + unsigned NumOfFields = Indices.size(); + bool IsTailGapOnly = NumOfFields > 1 && (NumOfFields == Indices.back() + 1); + if (IsTailGapOnly && + NumOfFields <= TLI->getMaxSupportedInterleaveFactor()) { + std::pair LT = getTypeLegalizationCost(FVTy); + if (LT.second.isVector() && + FVTy->getElementCount().isKnownMultipleOf(Factor)) { + auto *SubVecTy = VectorType::get( + FVTy->getElementType(), + FVTy->getElementCount().divideCoefficientBy(Factor)); + if (TLI->isLegalInterleavedAccessType(SubVecTy, NumOfFields, Alignment, + AddressSpace, DL)) { + // The cost is proportional to the total number of element accesses. + unsigned NumAccesses = getEstimatedVLFor(FVTy); + return NumAccesses * TTI::TCC_Basic; + } + } + } + } + InstructionCost MemCost = getMemoryOpCost(Opcode, VecTy, Alignment, AddressSpace, CostKind); unsigned VF = FVTy->getNumElements() / Factor; diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll index aeab26ebc9f44..2a3ba57bbe1ea 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-interleave.ll @@ -502,30 +502,23 @@ define void @store_factor_4_with_tail_gap(i32 %n, ptr noalias %a) { ; ; NO-VP-LABEL: @store_factor_4_with_tail_gap( ; NO-VP-NEXT: entry: -; NO-VP-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() -; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2 -; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], [[TMP1]] +; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], 8 ; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; NO-VP: vector.ph: -; NO-VP-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32() -; NO-VP-NEXT: [[TMP9:%.*]] = shl nuw i32 [[TMP2]], 2 -; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP9]] +; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 8 ; NO-VP-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]] -; NO-VP-NEXT: [[TMP7:%.*]] = call @llvm.stepvector.nxv4i32() -; NO-VP-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement poison, i32 [[TMP9]], i64 0 -; NO-VP-NEXT: [[BROADCAST_SPLAT3:%.*]] = shufflevector [[BROADCAST_SPLATINSERT2]], poison, zeroinitializer ; NO-VP-NEXT: br label [[VECTOR_BODY:%.*]] ; NO-VP: vector.body: ; NO-VP-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; NO-VP-NEXT: [[VEC_IND4:%.*]] = phi [ [[TMP7]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT5:%.*]], [[VECTOR_BODY]] ] -; NO-VP-NEXT: [[TMP11:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], [[VEC_IND4]], i32 0 -; NO-VP-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[VEC_IND4]], align 4 [[TMP11]], splat (i1 true)) -; NO-VP-NEXT: [[TMP12:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], [[VEC_IND4]], i32 1 -; NO-VP-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[VEC_IND4]], align 4 [[TMP12]], splat (i1 true)) -; NO-VP-NEXT: [[TMP13:%.*]] = getelementptr inbounds [4 x i32], ptr [[A]], [[VEC_IND4]], i32 2 -; NO-VP-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[VEC_IND4]], align 4 [[TMP13]], splat (i1 true)) -; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP9]] -; NO-VP-NEXT: [[VEC_IND_NEXT5]] = add nuw nsw [[VEC_IND4]], [[BROADCAST_SPLAT3]] +; NO-VP-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ , [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; NO-VP-NEXT: [[TMP0:%.*]] = getelementptr inbounds [4 x i32], ptr [[A:%.*]], i32 [[INDEX]], i32 0 +; NO-VP-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[VEC_IND]], <8 x i32> [[VEC_IND]], <16 x i32> +; NO-VP-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[VEC_IND]], <8 x i32> poison, <16 x i32> +; NO-VP-NEXT: [[TMP3:%.*]] = shufflevector <16 x i32> [[TMP1]], <16 x i32> [[TMP2]], <32 x i32> +; NO-VP-NEXT: [[INTERLEAVED_VEC:%.*]] = shufflevector <32 x i32> [[TMP3]], <32 x i32> poison, <32 x i32> +; NO-VP-NEXT: call void @llvm.masked.store.v32i32.p0(<32 x i32> [[INTERLEAVED_VEC]], ptr align 4 [[TMP0]], <32 x i1> ) +; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 +; NO-VP-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <8 x i32> [[VEC_IND]], splat (i32 8) ; NO-VP-NEXT: [[TMP14:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; NO-VP-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] ; NO-VP: middle.block: