Skip to content

AMDGPU: Cleanup the handling of flags in getTgtMemIntrinsic #179469

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nhaehnle merged 2 commits into from
Feb 23, 2026
Merged

AMDGPU: Cleanup the handling of flags in getTgtMemIntrinsic #179469

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bc9ae84
[CodeGen] Remove legacy getTgtMemIntrinsic overload
nhaehnle Jan 12, 2026
52c50d2
AMDGPU: Cleanup the handling of flags in getTgtMemIntrinsic
nhaehnle Feb 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 1 addition & 19 deletions llvm/include/llvm/CodeGen/TargetLowering.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1250,26 +1250,8 @@ class LLVM_ABI TargetLoweringBase {
/// access multiple memory locations.
virtual void getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
const CallBase &I, MachineFunction &MF,
unsigned Intrinsic) const {
// The default implementation forwards to the legacy single-info overload
// for compatibility.
IntrinsicInfo Info;
if (getTgtMemIntrinsic(Info, I, MF, Intrinsic))
Infos.push_back(Info);
}
unsigned Intrinsic) const {}

protected:
/// This is a legacy single-info overload. New code should override the
/// SmallVectorImpl overload instead to support multiple memory operands.
///
/// TODO: Remove this once the refactoring is complete.
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallBase &,
MachineFunction &,
unsigned /*Intrinsic*/) const {
return false;
}

public:
/// Returns true if the target can instruction select the specified FP
/// immediate natively. If false, the legalizer will materialize the FP
/// immediate as a load from a constant pool.
Expand Down
92 changes: 44 additions & 48 deletions llvm/lib/Target/AMDGPU/SIISelLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1346,13 +1346,12 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
const CallBase &CI,
MachineFunction &MF,
unsigned IntrID) const {
IntrinsicInfo Info;
Info.flags = MachineMemOperand::MONone;
MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
if (CI.hasMetadata(LLVMContext::MD_invariant_load))
Info.flags |= MachineMemOperand::MOInvariant;
Flags |= MachineMemOperand::MOInvariant;
if (CI.hasMetadata(LLVMContext::MD_nontemporal))
Info.flags |= MachineMemOperand::MONonTemporal;
Info.flags |= getTargetMMOFlags(CI);
Flags |= MachineMemOperand::MONonTemporal;
Flags |= getTargetMMOFlags(CI);

if (const AMDGPU::RsrcIntrinsic *RsrcIntr =
AMDGPU::lookupRsrcIntrinsic(IntrID)) {
Expand All @@ -1362,6 +1361,15 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
if (ME.doesNotAccessMemory())
return;

bool IsSPrefetch = IntrID == Intrinsic::amdgcn_s_buffer_prefetch_data;
if (!IsSPrefetch) {
Copy link
Copy Markdown
Contributor

@arsenm arsenm Feb 4, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this up here? Why isn't this just another case in the switch over intrinsic ID?

Copy link
Copy Markdown
Contributor Author

@nhaehnle nhaehnle Feb 4, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is part of the method that handles buffer/image intrinsics, and it does apply to all of them (except the prefetch one).

Copy link
Copy Markdown
Contributor Author

@nhaehnle nhaehnle Feb 4, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(I only moved that piece of code around a bit to be before the IntrinsicInfo declaration; in hindsight that code movement isn't too relevant.)

auto *Aux = cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1));
if (Aux->getZExtValue() & AMDGPU::CPol::VOLATILE)
Flags |= MachineMemOperand::MOVolatile;
}
Flags |= MachineMemOperand::MODereferenceable;

IntrinsicInfo Info;
// TODO: Should images get their own address space?
Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;

Expand All @@ -1384,14 +1392,6 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Info.ptrVal = RsrcArg;
}

bool IsSPrefetch = IntrID == Intrinsic::amdgcn_s_buffer_prefetch_data;
if (!IsSPrefetch) {
auto *Aux = cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1));
if (Aux->getZExtValue() & AMDGPU::CPol::VOLATILE)
Info.flags |= MachineMemOperand::MOVolatile;
}

Info.flags |= MachineMemOperand::MODereferenceable;
if (ME.onlyReadsMemory()) {
if (RsrcIntr->IsImage) {
unsigned MaxNumLanes = 4;
Expand All @@ -1414,7 +1414,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,

// FIXME: What does alignment mean for an image?
Info.opc = ISD::INTRINSIC_W_CHAIN;
Info.flags |= MachineMemOperand::MOLoad;
Info.flags = Flags | MachineMemOperand::MOLoad;
} else if (ME.onlyWritesMemory()) {
Info.opc = ISD::INTRINSIC_VOID;

Expand All @@ -1427,19 +1427,18 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
} else
Info.memVT = getValueType(MF.getDataLayout(), DataTy);

Info.flags |= MachineMemOperand::MOStore;
Info.flags = Flags | MachineMemOperand::MOStore;
} else {
// Atomic, NoReturn Sampler or prefetch
Info.opc = CI.getType()->isVoidTy() ? ISD::INTRINSIC_VOID
: ISD::INTRINSIC_W_CHAIN;
Info.flags |=
MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable;

if (!IsSPrefetch)
Info.flags |= MachineMemOperand::MOStore;

switch (IntrID) {
default:
Info.flags = Flags | MachineMemOperand::MOLoad;
if (!IsSPrefetch)
Info.flags |= MachineMemOperand::MOStore;

if ((RsrcIntr->IsImage && BaseOpcode->NoReturn) || IsSPrefetch) {
// Fake memory access type for no return sampler intrinsics
Info.memVT = MVT::i32;
Expand All @@ -1463,7 +1462,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
// Don't set an offset, since the pointer value always represents the
// base of the buffer.
Info.memVT = EVT::getIntegerVT(CI.getContext(), Width * 8);
Info.flags &= ~MachineMemOperand::MOStore;
Info.flags = Flags | MachineMemOperand::MOLoad;
Infos.push_back(Info);

// Entry 1: Store to LDS.
Expand All @@ -1475,8 +1474,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Info.offset = cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 2))
->getZExtValue();
Info.fallbackAddressSpace = AMDGPUAS::LOCAL_ADDRESS;
Info.flags &= ~MachineMemOperand::MOLoad;
Info.flags |= MachineMemOperand::MOStore;
Info.flags = Flags | MachineMemOperand::MOStore;
Infos.push_back(Info);
return;
}
Expand All @@ -1487,7 +1485,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Info.memVT =
memVTFromLoadIntrReturn(*this, MF.getDataLayout(), CI.getType(),
std::numeric_limits<unsigned>::max());
Info.flags &= ~MachineMemOperand::MOStore;
Info.flags = Flags | MachineMemOperand::MOLoad;
Infos.push_back(Info);
return;
}
Expand All @@ -1497,14 +1495,15 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
return;
}

IntrinsicInfo Info;
switch (IntrID) {
case Intrinsic::amdgcn_ds_ordered_add:
case Intrinsic::amdgcn_ds_ordered_swap: {
Info.opc = ISD::INTRINSIC_W_CHAIN;
Info.memVT = MVT::getVT(CI.getType());
Info.ptrVal = CI.getOperand(0);
Info.align.reset();
Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;

const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(4));
if (!Vol->isZero())
Expand All @@ -1529,7 +1528,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Info.memVT = MVT::getVT(CI.getType());
Info.ptrVal = CI.getOperand(0);
Info.align.reset();
Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;

const ConstantInt *Vol = cast<ConstantInt>(CI.getOperand(1));
if (!Vol->isZero())
Expand All @@ -1548,7 +1547,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Info.memVT = MVT::i64;
Info.size = 8;
Info.align.reset();
Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
Infos.push_back(Info);
return;
}
Expand All @@ -1564,8 +1563,8 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,

Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE;
Info.align.reset();
Info.flags |=
MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable;
Info.flags = Flags | MachineMemOperand::MOLoad |
MachineMemOperand::MODereferenceable;
Infos.push_back(Info);
return;
}
Expand All @@ -1578,9 +1577,9 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Info.memVT = MVT::getVT(CI.getType());
Info.ptrVal = CI.getOperand(0);
Info.align.reset();
Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
MachineMemOperand::MODereferenceable |
MachineMemOperand::MOVolatile;
Info.flags =
Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
MachineMemOperand::MODereferenceable | MachineMemOperand::MOVolatile;
Infos.push_back(Info);
return;
}
Expand All @@ -1603,7 +1602,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Info.memVT = MVT::getVT(CI.getType());
Info.ptrVal = CI.getOperand(0);
Info.align.reset();
Info.flags |= MachineMemOperand::MOLoad;
Info.flags = Flags | MachineMemOperand::MOLoad;
Infos.push_back(Info);
return;
}
Expand Down Expand Up @@ -1669,9 +1668,9 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Info.align = Align(4);

if (IntrID == Intrinsic::amdgcn_ds_gws_barrier)
Info.flags |= MachineMemOperand::MOLoad;
Info.flags = Flags | MachineMemOperand::MOLoad;
else
Info.flags |= MachineMemOperand::MOStore;
Info.flags = Flags | MachineMemOperand::MOStore;
Infos.push_back(Info);
return;
}
Expand All @@ -1688,12 +1687,11 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Info.memVT = EVT::getIntegerVT(CI.getContext(), getIntrMemWidth(IntrID));
Info.ptrVal = CI.getArgOperand(0); // Global pointer
Info.offset = cast<ConstantInt>(CI.getArgOperand(2))->getSExtValue();
Info.flags |= MachineMemOperand::MOLoad;
Info.flags = Flags | MachineMemOperand::MOLoad;
Infos.push_back(Info);

// Entry 1: Store to LDS (same offset).
Info.flags &= ~MachineMemOperand::MOLoad;
Info.flags |= MachineMemOperand::MOStore;
Info.flags = Flags | MachineMemOperand::MOStore;
Info.ptrVal = CI.getArgOperand(1); // LDS pointer
Infos.push_back(Info);
return;
Expand All @@ -1707,12 +1705,11 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Info.memVT = EVT::getIntegerVT(CI.getContext(), getIntrMemWidth(IntrID));
Info.ptrVal = CI.getArgOperand(1); // LDS pointer
Info.offset = cast<ConstantInt>(CI.getArgOperand(2))->getSExtValue();
Info.flags |= MachineMemOperand::MOLoad;
Info.flags = Flags | MachineMemOperand::MOLoad;
Infos.push_back(Info);

// Entry 1: Store to global (same offset).
Info.flags &= ~MachineMemOperand::MOLoad;
Info.flags |= MachineMemOperand::MOStore;
Info.flags = Flags | MachineMemOperand::MOStore;
Info.ptrVal = CI.getArgOperand(0); // Global pointer
Infos.push_back(Info);
return;
Expand All @@ -1724,15 +1721,15 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
unsigned Width = cast<ConstantInt>(CI.getArgOperand(2))->getZExtValue();
auto *Aux = cast<ConstantInt>(CI.getArgOperand(CI.arg_size() - 1));
bool IsVolatile = Aux->getZExtValue() & AMDGPU::CPol::VOLATILE;
if (IsVolatile)
Flags |= MachineMemOperand::MOVolatile;

// Entry 0: Load from source (global/flat).
Info.opc = ISD::INTRINSIC_VOID;
Info.memVT = EVT::getIntegerVT(CI.getContext(), Width * 8);
Info.ptrVal = CI.getArgOperand(0); // Source pointer
Info.offset = cast<ConstantInt>(CI.getArgOperand(3))->getSExtValue();
Info.flags |= MachineMemOperand::MOLoad;
if (IsVolatile)
Info.flags |= MachineMemOperand::MOVolatile;
Info.flags = Flags | MachineMemOperand::MOLoad;
Infos.push_back(Info);

// Entry 1: Store to LDS.
Expand All @@ -1741,8 +1738,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Info.memVT = EVT::getIntegerVT(CI.getContext(),
Width * 8 * Subtarget->getWavefrontSize());
Info.ptrVal = CI.getArgOperand(1); // LDS destination pointer
Info.flags &= ~MachineMemOperand::MOLoad;
Info.flags |= MachineMemOperand::MOStore;
Info.flags = Flags | MachineMemOperand::MOStore;
Infos.push_back(Info);
return;
}
Expand All @@ -1763,7 +1759,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Info.size = 4;
Info.align = Align(4);

Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
Info.flags = Flags | MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
Infos.push_back(Info);
return;
}
Expand All @@ -1773,7 +1769,7 @@ void SITargetLowering::getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
Info.opc = ISD::INTRINSIC_VOID;
Info.memVT = EVT::getIntegerVT(CI.getContext(), 8);
Info.ptrVal = CI.getArgOperand(0);
Info.flags |= MachineMemOperand::MOLoad;
Info.flags = Flags | MachineMemOperand::MOLoad;
Infos.push_back(Info);
return;
}
Expand Down
Loading