From 6ed1673c844e3f91b613cad2722c16a3e4c4146e Mon Sep 17 00:00:00 2001 From: ShihPo Hung Date: Thu, 13 Nov 2025 23:43:59 -0800 Subject: [PATCH 1/5] [TTI] Use MemIntrinsicCostAttributes for getMaskedMemoryOpCost - Split from #165532. This is a step toward a unified interface for masked/gather-scatter/strided/expand-compress cost modeling. - Replace the ad-hoc parameter list with a single attributes object. API change: // Before: InstructionCost getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind); // After: InstructionCost getMaskedMemoryOpCost(MemIntrinsicCostAttributes, CostKind); Notes: - NFCI intended: callers populate MemIntrinsicCostAttributes with the same information as before. - Follow-up: migrate gather/scatter, strided, and expand/compress cost queries to the same attributes-based entry point. --- .../llvm/Analysis/TargetTransformInfo.h | 49 ++++++++++++++++++- .../llvm/Analysis/TargetTransformInfoImpl.h | 3 +- llvm/include/llvm/CodeGen/BasicTTIImpl.h | 24 +++++---- llvm/lib/Analysis/TargetTransformInfo.cpp | 5 +- .../AArch64/AArch64TargetTransformInfo.cpp | 8 +-- .../AArch64/AArch64TargetTransformInfo.h | 3 +- .../lib/Target/ARM/ARMTargetTransformInfo.cpp | 14 +++--- llvm/lib/Target/ARM/ARMTargetTransformInfo.h | 3 +- .../Hexagon/HexagonTargetTransformInfo.cpp | 6 +-- .../Hexagon/HexagonTargetTransformInfo.h | 3 +- .../Target/RISCV/RISCVTargetTransformInfo.cpp | 12 +++-- .../Target/RISCV/RISCVTargetTransformInfo.h | 3 +- .../lib/Target/X86/X86TargetTransformInfo.cpp | 19 ++++--- llvm/lib/Target/X86/X86TargetTransformInfo.h | 3 +- .../Transforms/Vectorize/LoopVectorize.cpp | 6 ++- .../Transforms/Vectorize/SLPVectorizer.cpp | 17 ++++--- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 8 +-- 17 files changed, 124 insertions(+), 62 deletions(-) diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h index 0f17312b03827..a5b77a76fd92d 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -123,6 +123,53 @@ struct HardwareLoopInfo { LLVM_ABI bool canAnalyze(LoopInfo &LI); }; +/// Information for memory intrinsic cost model. +class MemIntrinsicCostAttributes { + /// Optional original context instruction, if one exists, e.g. the + /// load/store to transform to the intrinsic. + const Instruction *I = nullptr; + + /// Address in memory. + const Value *Ptr = nullptr; + + /// Vector type of the data to be loaded or stored. + Type *DataTy = nullptr; + Intrinsic::ID IID; + + /// True when the memory access is predicated with a mask + /// that is not a compile-time constant. + bool VariableMask = true; + unsigned AddressSpace = 0; + + /// Alignment of single element. + Align Alignment; + +public: + LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy, + const Value *Ptr, bool VariableMask, + Align Alignment, + const Instruction *I = nullptr) + : I(I), Ptr(Ptr), DataTy(DataTy), IID(Id), VariableMask(VariableMask), + Alignment(Alignment) {} + LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy, + Align Alignment, unsigned AddressSpace) + : DataTy(DataTy), IID(Id), AddressSpace(AddressSpace), + Alignment(Alignment) {} + LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy, + bool VariableMask, Align Alignment, + const Instruction *I = nullptr) + : I(I), DataTy(DataTy), IID(Id), VariableMask(VariableMask), + Alignment(Alignment) {} + + Intrinsic::ID getID() const { return IID; } + const Instruction *getInst() const { return I; } + const Value *getPointer() const { return Ptr; } + Type *getDataType() const { return DataTy; } + bool getVariableMask() const { return VariableMask; } + unsigned getAddressSpace() const { return AddressSpace; } + Align getAlignment() const { return Alignment; } +}; + class IntrinsicCostAttributes { const IntrinsicInst *II = nullptr; Type *RetTy = nullptr; @@ -1556,7 +1603,7 @@ class TargetTransformInfo { /// \return The cost of masked Load and Store instructions. LLVM_ABI InstructionCost getMaskedMemoryOpCost( - unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, + const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const; /// \return The cost of Gather or Scatter operation diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index aacb88d2f9684..d8e35748f53e5 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -842,8 +842,7 @@ class TargetTransformInfoImplBase { } virtual InstructionCost - getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, - unsigned AddressSpace, + getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const { return 1; } diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h index e8dbc964a943e..b74501040298a 100644 --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -1547,9 +1547,13 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase { } InstructionCost - getMaskedMemoryOpCost(unsigned Opcode, Type *DataTy, Align Alignment, - unsigned AddressSpace, + getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const override { + Type *DataTy = MICA.getDataType(); + Align Alignment = MICA.getAlignment(); + unsigned Opcode = MICA.getID() == Intrinsic::masked_load + ? Instruction::Load + : Instruction::Store; // TODO: Pass on AddressSpace when we have test coverage. return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, true, false, CostKind); @@ -1606,10 +1610,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase { // Firstly, the cost of load/store operation. InstructionCost Cost; - if (UseMaskForCond || UseMaskForGaps) - Cost = thisT()->getMaskedMemoryOpCost(Opcode, VecTy, Alignment, - AddressSpace, CostKind); - else + if (UseMaskForCond || UseMaskForGaps) { + unsigned IID = Opcode == Instruction::Load ? Intrinsic::masked_load + : Intrinsic::masked_store; + Cost = thisT()->getMaskedMemoryOpCost( + {IID, VecTy, Alignment, AddressSpace}, CostKind); + } else Cost = thisT()->getMemoryOpCost(Opcode, VecTy, Alignment, AddressSpace, CostKind); @@ -2408,14 +2414,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase { case Intrinsic::masked_store: { Type *Ty = Tys[0]; Align TyAlign = thisT()->DL.getABITypeAlign(Ty); - return thisT()->getMaskedMemoryOpCost(Instruction::Store, Ty, TyAlign, 0, - CostKind); + return thisT()->getMaskedMemoryOpCost({IID, Ty, TyAlign, 0}, CostKind); } case Intrinsic::masked_load: { Type *Ty = RetTy; Align TyAlign = thisT()->DL.getABITypeAlign(Ty); - return thisT()->getMaskedMemoryOpCost(Instruction::Load, Ty, TyAlign, 0, - CostKind); + return thisT()->getMaskedMemoryOpCost({IID, Ty, TyAlign, 0}, CostKind); } case Intrinsic::experimental_vp_strided_store: { auto *Ty = cast(ICA.getArgTypes()[0]); diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp index 0426ac7e62fab..45369f0ffe137 100644 --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -1183,10 +1183,9 @@ InstructionCost TargetTransformInfo::getMemoryOpCost( } InstructionCost TargetTransformInfo::getMaskedMemoryOpCost( - unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, + const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const { - InstructionCost Cost = TTIImpl->getMaskedMemoryOpCost(Opcode, Src, Alignment, - AddressSpace, CostKind); + InstructionCost Cost = TTIImpl->getMaskedMemoryOpCost(MICA, CostKind); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index 10f2c80edc1b3..9dbca001a3ff6 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -4720,12 +4720,12 @@ bool AArch64TTIImpl::prefersVectorizedAddressing() const { } InstructionCost -AArch64TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, - Align Alignment, unsigned AddressSpace, +AArch64TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const { + Type *Src = MICA.getDataType(); + if (useNeonVector(Src)) - return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace, - CostKind); + return BaseT::getMaskedMemoryOpCost(MICA, CostKind); auto LT = getTypeLegalizationCost(Src); if (!LT.first.isValid()) return InstructionCost::getInvalid(); diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h index e3b0a1bec53ec..e04a5522e392c 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -188,8 +188,7 @@ class AArch64TTIImpl final : public BasicTTIImplBase { unsigned Opcode2) const; InstructionCost - getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, - unsigned AddressSpace, + getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const override; InstructionCost diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp index 9b250e6cac3ab..9b80c4f95ccff 100644 --- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp +++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp @@ -1631,20 +1631,22 @@ InstructionCost ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, } InstructionCost -ARMTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, - unsigned AddressSpace, +ARMTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const { + unsigned IID = MICA.getID(); + Type *Src = MICA.getDataType(); + Align Alignment = MICA.getAlignment(); + unsigned AddressSpace = MICA.getAddressSpace(); if (ST->hasMVEIntegerOps()) { - if (Opcode == Instruction::Load && + if (IID == Intrinsic::masked_load && isLegalMaskedLoad(Src, Alignment, AddressSpace)) return ST->getMVEVectorCostFactor(CostKind); - if (Opcode == Instruction::Store && + if (IID == Intrinsic::masked_store && isLegalMaskedStore(Src, Alignment, AddressSpace)) return ST->getMVEVectorCostFactor(CostKind); } if (!isa(Src)) - return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace, - CostKind); + return BaseT::getMaskedMemoryOpCost(MICA, CostKind); // Scalar cost, which is currently very high due to the efficiency of the // generated code. return cast(Src)->getNumElements() * 8; diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h index 0810c5532ed91..919a6fc9fd0b0 100644 --- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h +++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h @@ -275,8 +275,7 @@ class ARMTTIImpl final : public BasicTTIImplBase { const Instruction *I = nullptr) const override; InstructionCost - getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, - unsigned AddressSpace, + getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const override; InstructionCost getInterleavedMemoryOpCost( diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp index e925e041eb64e..8f3f0cc8abb01 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp +++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp @@ -224,11 +224,9 @@ InstructionCost HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, } InstructionCost -HexagonTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, - Align Alignment, unsigned AddressSpace, +HexagonTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const { - return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace, - CostKind); + return BaseT::getMaskedMemoryOpCost(MICA, CostKind); } InstructionCost diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h index cec2bf9656ffc..e95b5a10b76a7 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h +++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h @@ -120,8 +120,7 @@ class HexagonTTIImpl final : public BasicTTIImplBase { TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None}, const Instruction *I = nullptr) const override; InstructionCost - getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, - unsigned AddressSpace, + getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const override; InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index 7bc0b5b394828..85d66392862d9 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -1001,13 +1001,17 @@ InstructionCost RISCVTTIImpl::getScalarizationOverhead( } InstructionCost -RISCVTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, - unsigned AddressSpace, +RISCVTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const { + unsigned Opcode = MICA.getID() == Intrinsic::masked_load ? Instruction::Load + : Instruction::Store; + Type *Src = MICA.getDataType(); + Align Alignment = MICA.getAlignment(); + unsigned AddressSpace = MICA.getAddressSpace(); + if (!isLegalMaskedLoadStore(Src, Alignment) || CostKind != TTI::TCK_RecipThroughput) - return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace, - CostKind); + return BaseT::getMaskedMemoryOpCost(MICA, CostKind); return getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind); } diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h index 6886e8964e29e..39c1173e2986c 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h @@ -144,8 +144,7 @@ class RISCVTTIImpl final : public BasicTTIImplBase { bool shouldConsiderVectorizationRegPressure() const override { return true; } InstructionCost - getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, - unsigned AddressSpace, + getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const override; InstructionCost diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index 0b1430e373fc7..4b77bf925b2ba 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -5411,9 +5411,14 @@ InstructionCost X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, } InstructionCost -X86TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *SrcTy, Align Alignment, - unsigned AddressSpace, +X86TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const { + unsigned Opcode = MICA.getID() == Intrinsic::masked_load ? Instruction::Load + : Instruction::Store; + Type *SrcTy = MICA.getDataType(); + Align Alignment = MICA.getAlignment(); + unsigned AddressSpace = MICA.getAddressSpace(); + bool IsLoad = (Instruction::Load == Opcode); bool IsStore = (Instruction::Store == Opcode); @@ -6647,10 +6652,12 @@ InstructionCost X86TTIImpl::getInterleavedMemoryOpCostAVX512( LegalVT.getVectorNumElements()); InstructionCost MemOpCost; bool UseMaskedMemOp = UseMaskForCond || UseMaskForGaps; - if (UseMaskedMemOp) - MemOpCost = getMaskedMemoryOpCost(Opcode, SingleMemOpTy, Alignment, - AddressSpace, CostKind); - else + if (UseMaskedMemOp) { + unsigned IID = Opcode == Instruction::Load ? Intrinsic::masked_load + : Intrinsic::masked_store; + MemOpCost = getMaskedMemoryOpCost( + {IID, SingleMemOpTy, Alignment, AddressSpace}, CostKind); + } else MemOpCost = getMemoryOpCost(Opcode, SingleMemOpTy, Alignment, AddressSpace, CostKind); diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h index de5e1c297b1e4..df1393ce16ca1 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.h +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h @@ -183,8 +183,7 @@ class X86TTIImpl final : public BasicTTIImplBase { TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None}, const Instruction *I = nullptr) const override; InstructionCost - getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, - unsigned AddressSpace, + getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const override; InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index e5c3f17860103..dd984c485c1ee 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -5197,8 +5197,10 @@ LoopVectorizationCostModel::getConsecutiveMemOpCost(Instruction *I, const Align Alignment = getLoadStoreAlignment(I); InstructionCost Cost = 0; if (Legal->isMaskRequired(I)) { - Cost += TTI.getMaskedMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS, - CostKind); + unsigned IID = I->getOpcode() == Instruction::Load + ? Intrinsic::masked_load + : Intrinsic::masked_store; + Cost += TTI.getMaskedMemoryOpCost({IID, VectorTy, Alignment, AS}, CostKind); } else { TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(I->getOperand(0)); Cost += TTI.getMemoryOpCost(I->getOpcode(), VectorTy, Alignment, AS, diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index bf3f52c51b64c..f7a800652c551 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -6848,9 +6848,10 @@ static bool isMaskedLoadCompress( ScalarLoadsCost; InstructionCost LoadCost = 0; if (IsMasked) { - LoadCost = - TTI.getMaskedMemoryOpCost(Instruction::Load, LoadVecTy, CommonAlignment, - LI->getPointerAddressSpace(), CostKind); + LoadCost = TTI.getMaskedMemoryOpCost({Intrinsic::masked_load, LoadVecTy, + CommonAlignment, + LI->getPointerAddressSpace()}, + CostKind); } else { LoadCost = TTI.getMemoryOpCost(Instruction::Load, LoadVecTy, CommonAlignment, @@ -7249,8 +7250,9 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads( break; case LoadsState::CompressVectorize: VecLdCost += TTI.getMaskedMemoryOpCost( - Instruction::Load, SubVecTy, CommonAlignment, - LI0->getPointerAddressSpace(), CostKind) + + {Intrinsic::masked_load, SubVecTy, CommonAlignment, + LI0->getPointerAddressSpace()}, + CostKind) + VectorGEPCost + ::getShuffleCost(TTI, TTI::SK_PermuteSingleSrc, SubVecTy, {}, CostKind); @@ -15041,8 +15043,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, CommonAlignment, LI0->getPointerAddressSpace(), CostKind); } else if (IsMasked) { VecLdCost = TTI->getMaskedMemoryOpCost( - Instruction::Load, LoadVecTy, CommonAlignment, - LI0->getPointerAddressSpace(), CostKind); + {Intrinsic::masked_load, LoadVecTy, CommonAlignment, + LI0->getPointerAddressSpace()}, + CostKind); // TODO: include this cost into CommonCost. VecLdCost += ::getShuffleCost(*TTI, TTI::SK_PermuteSingleSrc, LoadVecTy, CompressMask, CostKind); diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 1ee405a62aa68..aa84d68be729b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -3566,8 +3566,10 @@ InstructionCost VPWidenMemoryRecipe::computeCost(ElementCount VF, InstructionCost Cost = 0; if (IsMasked) { + unsigned IID = isa(this) ? Intrinsic::masked_load + : Intrinsic::masked_store; Cost += - Ctx.TTI.getMaskedMemoryOpCost(Opcode, Ty, Alignment, AS, Ctx.CostKind); + Ctx.TTI.getMaskedMemoryOpCost({IID, Ty, Alignment, AS}, Ctx.CostKind); } else { TTI::OperandValueInfo OpInfo = Ctx.getOperandInfo( isa(this) ? getOperand(0) @@ -3686,7 +3688,7 @@ InstructionCost VPWidenLoadEVLRecipe::computeCost(ElementCount VF, unsigned AS = cast(Ctx.Types.inferScalarType(getAddr())) ->getAddressSpace(); InstructionCost Cost = Ctx.TTI.getMaskedMemoryOpCost( - Instruction::Load, Ty, Alignment, AS, Ctx.CostKind); + {Intrinsic::masked_load, Ty, Alignment, AS}, Ctx.CostKind); if (!Reverse) return Cost; @@ -3795,7 +3797,7 @@ InstructionCost VPWidenStoreEVLRecipe::computeCost(ElementCount VF, unsigned AS = cast(Ctx.Types.inferScalarType(getAddr())) ->getAddressSpace(); InstructionCost Cost = Ctx.TTI.getMaskedMemoryOpCost( - Instruction::Store, Ty, Alignment, AS, Ctx.CostKind); + {Intrinsic::masked_store, Ty, Alignment, AS}, Ctx.CostKind); if (!Reverse) return Cost; From 5c53a465a32ae9d6edd8d16bd407d78428ea2999 Mon Sep 17 00:00:00 2001 From: ShihPo Hung Date: Fri, 14 Nov 2025 06:25:39 -0800 Subject: [PATCH 2/5] Address comments --- llvm/include/llvm/Analysis/TargetTransformInfo.h | 2 +- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h index a5b77a76fd92d..b2ed523c89aed 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -125,7 +125,7 @@ struct HardwareLoopInfo { /// Information for memory intrinsic cost model. class MemIntrinsicCostAttributes { - /// Optional original context instruction, if one exists, e.g. the + /// Optional context instruction, if one exists, e.g. the /// load/store to transform to the intrinsic. const Instruction *I = nullptr; diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index aa84d68be729b..7ce60bb5bbaa4 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -3688,7 +3688,7 @@ InstructionCost VPWidenLoadEVLRecipe::computeCost(ElementCount VF, unsigned AS = cast(Ctx.Types.inferScalarType(getAddr())) ->getAddressSpace(); InstructionCost Cost = Ctx.TTI.getMaskedMemoryOpCost( - {Intrinsic::masked_load, Ty, Alignment, AS}, Ctx.CostKind); + {Intrinsic::vp_load, Ty, Alignment, AS}, Ctx.CostKind); if (!Reverse) return Cost; From 41c8506e545199534d90bfd52407fc583d51f652 Mon Sep 17 00:00:00 2001 From: ShihPo Hung Date: Sun, 16 Nov 2025 18:36:07 -0800 Subject: [PATCH 3/5] Keep subset of MICA --- .../llvm/Analysis/TargetTransformInfo.h | 27 +++---------------- 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h index b2ed523c89aed..a65e4667ab76c 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -125,47 +125,26 @@ struct HardwareLoopInfo { /// Information for memory intrinsic cost model. class MemIntrinsicCostAttributes { - /// Optional context instruction, if one exists, e.g. the - /// load/store to transform to the intrinsic. - const Instruction *I = nullptr; - - /// Address in memory. - const Value *Ptr = nullptr; - /// Vector type of the data to be loaded or stored. Type *DataTy = nullptr; + + /// ID of the memory intrinsic. Intrinsic::ID IID; - /// True when the memory access is predicated with a mask - /// that is not a compile-time constant. - bool VariableMask = true; + /// Address space of the pointer. unsigned AddressSpace = 0; /// Alignment of single element. Align Alignment; public: - LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy, - const Value *Ptr, bool VariableMask, - Align Alignment, - const Instruction *I = nullptr) - : I(I), Ptr(Ptr), DataTy(DataTy), IID(Id), VariableMask(VariableMask), - Alignment(Alignment) {} LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy, Align Alignment, unsigned AddressSpace) : DataTy(DataTy), IID(Id), AddressSpace(AddressSpace), Alignment(Alignment) {} - LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy, - bool VariableMask, Align Alignment, - const Instruction *I = nullptr) - : I(I), DataTy(DataTy), IID(Id), VariableMask(VariableMask), - Alignment(Alignment) {} Intrinsic::ID getID() const { return IID; } - const Instruction *getInst() const { return I; } - const Value *getPointer() const { return Ptr; } Type *getDataType() const { return DataTy; } - bool getVariableMask() const { return VariableMask; } unsigned getAddressSpace() const { return AddressSpace; } Align getAlignment() const { return Alignment; } }; From d44b7e2a7a75c7913d282e765c8103f8624c001a Mon Sep 17 00:00:00 2001 From: ShihPo Hung Date: Sun, 16 Nov 2025 18:40:33 -0800 Subject: [PATCH 4/5] Add FIXME to VPlanRecipes --- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 7ce60bb5bbaa4..373aae57d7ebc 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -3687,8 +3687,10 @@ InstructionCost VPWidenLoadEVLRecipe::computeCost(ElementCount VF, Type *Ty = toVectorTy(getLoadStoreType(&Ingredient), VF); unsigned AS = cast(Ctx.Types.inferScalarType(getAddr())) ->getAddressSpace(); + // FIXME: getMaskedMemoryOpCost assumes masked_* intrinsics. + // After migrating to getMemIntrinsicInstrCost, switch this to vp_load. InstructionCost Cost = Ctx.TTI.getMaskedMemoryOpCost( - {Intrinsic::vp_load, Ty, Alignment, AS}, Ctx.CostKind); + {Intrinsic::masked_load, Ty, Alignment, AS}, Ctx.CostKind); if (!Reverse) return Cost; @@ -3796,6 +3798,8 @@ InstructionCost VPWidenStoreEVLRecipe::computeCost(ElementCount VF, Type *Ty = toVectorTy(getLoadStoreType(&Ingredient), VF); unsigned AS = cast(Ctx.Types.inferScalarType(getAddr())) ->getAddressSpace(); + // FIXME: getMaskedMemoryOpCost assumes masked_* intrinsics. + // After migrating to getMemIntrinsicInstrCost, switch this to vp_load. InstructionCost Cost = Ctx.TTI.getMaskedMemoryOpCost( {Intrinsic::masked_store, Ty, Alignment, AS}, Ctx.CostKind); if (!Reverse) From dddfdc9aca0e70ab484db43113baaeecc6e54247 Mon Sep 17 00:00:00 2001 From: ShihPo Hung Date: Sun, 16 Nov 2025 22:57:36 -0800 Subject: [PATCH 5/5] Fix typo --- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 373aae57d7ebc..f7604e59725b7 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -3799,7 +3799,7 @@ InstructionCost VPWidenStoreEVLRecipe::computeCost(ElementCount VF, unsigned AS = cast(Ctx.Types.inferScalarType(getAddr())) ->getAddressSpace(); // FIXME: getMaskedMemoryOpCost assumes masked_* intrinsics. - // After migrating to getMemIntrinsicInstrCost, switch this to vp_load. + // After migrating to getMemIntrinsicInstrCost, switch this to vp_store. InstructionCost Cost = Ctx.TTI.getMaskedMemoryOpCost( {Intrinsic::masked_store, Ty, Alignment, AS}, Ctx.CostKind); if (!Reverse)