Skip to content

[rocjitsu] Preserve atomic semantics and GLOBAL-only decoding #11489

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kuhar merged 2 commits into from
Sep 12, 2026
Merged

[rocjitsu] Preserve atomic semantics and GLOBAL-only decoding #11489

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e4908da
[rocjitsu] Preserve floating atomic policies and compare-store operands
kuhar Sep 10, 2026
d3cee1f
[rocjitsu] Preserve atomic semantics and GLOBAL-only decoding
kuhar Sep 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 57 additions & 11 deletions emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8036,11 +8036,7 @@ def _append_global_addtid_addresses(self, L: list[str]) -> None:
L.append(' d->wf_size = wf.wf_size();')
L.append(' d->wg_id = wf.wg_id(); d->wf_id = wf.wf_id();')
L.append(' uint64_t base = amdgpu::RegisterAccess(wf).read_scalar64(saddr);')
offset_expr = (
'signed_ioffset(inst_.ioffset)'
if self.isa_spec.arch_name == 'cdna5'
else 'static_cast<int32_t>(inst_.ioffset << 8) >> 8'
)
offset_expr = self.isa_spec.profile.global_addtid_offset_expr
L.append(f' int64_t offset = static_cast<int64_t>({offset_expr});')
L.append(' for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) {')
L.append(' if (!(exec & (1ULL << lane))) continue;')
Expand Down Expand Up @@ -8084,7 +8080,9 @@ def _gen_global_store_addtid(
self._append_global_addtid_addresses(L)
L.append(' auto &cu = wf.cu();')
L.append(' uint64_t exec = wf.exec();')
L.append(f" uint32_t data_base = {self._vgpr_base_expr('vsrc')};")
L.append(
f" uint32_t data_base = {self._vgpr_base_expr(self.isa_spec.profile.flat_store_src_field)};"
)
L.append(' d->store_data.resize(wf.wf_size() * 4);')
L.append(' for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) {')
L.append(' if (!(exec & (1ULL << lane))) continue;')
Expand All @@ -8099,9 +8097,13 @@ def _gen_global_store_addtid(
_ATOMIC_OP_ENUM: dict[str, str] = {
'swap': 'amdgpu::AtomicOp::SWAP',
'cmpswap': 'amdgpu::AtomicOp::CMPSWAP',
'condxchg32': 'amdgpu::AtomicOp::CONDXCHG32',
'fcmpswap': 'amdgpu::AtomicOp::FCMPSWAP',
'mskor': 'amdgpu::AtomicOp::MSKOR',
'add': 'amdgpu::AtomicOp::ADD',
'sub': 'amdgpu::AtomicOp::SUB',
'sub_clamp': 'amdgpu::AtomicOp::SUB_CLAMP',
'cond_sub': 'amdgpu::AtomicOp::COND_SUB',
'rsub': 'amdgpu::AtomicOp::RSUB',
'smin': 'amdgpu::AtomicOp::SMIN',
'umin': 'amdgpu::AtomicOp::UMIN',
Expand All @@ -8127,14 +8129,30 @@ def _buffer_payload_exec_expr(self) -> str:
return 'd->exec_mask'
return 'wf.exec()'

def _append_atomic_fp_policy(
self, lines: list[str], sem: InstructionSemantics, *, ds: bool
) -> None:
"""Carry manual-defined scalar FP policies through deferred memory execution."""
if sem.operation not in ('fadd', 'fmin', 'fmax', 'fcmpswap'):
return
profile = self.isa_spec.profile
memory_mode, lds_mode = profile.scalar_atomic_denorm_modes(
sem.operation, sem.elem_size, ds=ds
)
lines.append(f' d->atomic_denorm_mode = {memory_mode};')
lines.append(f' d->atomic_lds_denorm_mode = {lds_mode};')
lines.append(
f' d->atomic_legacy_minmax = {str(profile.atomic_legacy_minmax).lower()};'
)

def _gen_flat_atomic(
self, dst: list[str], src: list[str], sem: InstructionSemantics
) -> str:
"""Generate flat_atomic execute() body.

If the operation is recognized, emits a full VectorMemState setup
with AtomicOp for the pipeline. Unrecognized variants (FP atomics,
etc.) fall back to a TODO stub.
with AtomicOp for the pipeline. Unrecognized variants raise an
explicit unimplemented-instruction error.
"""
if sem.operation is None or sem.operation not in self._ATOMIC_OP_ENUM:
return f' (void)wf;\n throw util::UnimplementedInst(mnemonic()); // TODO: unhandled flat_atomic variant ({sem.name})'
Expand All @@ -8154,6 +8172,7 @@ def _gen_flat_atomic(
L.append(' d->num_elems = 1;')
L.append(f' d->is_load = {self._atomic_return_expr(sc0)};')
L.append(f' d->atomic_op = {op_enum};')
self._append_atomic_fp_policy(L, sem, ds=False)
self._append_wait_counter_type(L, 'flat_atomic')
L.append(f' d->mtype = {self._mtype_expr()};')
L.append(f' d->non_temporal = {nt};')
Expand Down Expand Up @@ -8201,6 +8220,7 @@ def _gen_buffer_atomic(
L.append(' d->num_elems = 1;')
L.append(f' d->is_load = {self._atomic_return_expr(sc0)};')
L.append(f' d->atomic_op = {op_enum};')
self._append_atomic_fp_policy(L, sem, ds=False)
self._append_wait_counter_type(L, 'buffer_atomic')
L.append(f' d->mtype = {self._mtype_expr()};')
L.append(f' d->non_temporal = {nt};')
Expand Down Expand Up @@ -8237,7 +8257,7 @@ def _gen_ds_atomic(
returns_data = 'vdst' in dst

L = []
is_cmpswap = sem.operation == 'cmpswap'
is_cmpswap = sem.operation in ('cmpswap', 'fcmpswap')
L.append(
' auto d = std::make_unique<amdgpu::VectorMemState>(amdgpu::LOCAL_MEM);'
)
Expand All @@ -8247,11 +8267,19 @@ def _gen_ds_atomic(
L.append(' d->num_elems = 1;')
L.append(f' d->is_load = {str(returns_data).lower()};')
L.append(f' d->atomic_op = {op_enum};')
self._append_atomic_fp_policy(L, sem, ds=True)
if sem.operation in ('pk_add_f16', 'pk_add_bf16'):
# CDNA5 ISA 12.2 groups packed F16/BF16 under DS denorm_double controls.
L.append(' d->packed_denorm_mode = wf.fp_denorm_mode_f16_f64();')
self._append_wait_counter_type(L, 'ds_atomic')
L.append(' ds_calculate_addresses(inst_, wf, *d);')
if sem.operation == 'condxchg32':
# Conditional exchange uses a qword-aligned 16-bit LDS address.
L.append(' for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) {')
L.append(' d->per_lane_addr[lane] = wf.lds_base() +')
L.append(' ((d->per_lane_addr[lane] - wf.lds_base()) & 0xfff8u);')
L.append(' }')

L.append(' auto &cu = wf.cu();')
L.append(' uint64_t exec = wf.exec();')
L.append(
Expand All @@ -8261,6 +8289,9 @@ def _gen_ds_atomic(
L.append(
f" uint32_t data1_base = {self._vgpr_base_expr('data1', role='Src2')};"
)
if is_cmpswap and self.isa_spec.profile.ds_compare_store_compare_first:
# Normalize older DS comparison/replacement order to the pipeline contract.
L.append(' std::swap(data_base, data1_base);')
stride = data_dwords * 4
L.append(f' d->store_data.resize(wf.wf_size() * {stride});')
L.append(' for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) {')
Expand Down Expand Up @@ -9711,6 +9742,14 @@ def gen_insts(self) -> None:
cdna5_swmmac_has_modifiers = self._cdna5_swmmac_has_modifiers(inst)
operand_size_exprs: dict[str, str] = {}
for opnd in inst.operands:
# FLAT's optional scalar address is constructed below,
# including when a GLOBAL-only XML form lists it explicitly.
if (
enc.enc_name == 'ENC_FLAT'
and opnd.name == 'saddr'
and any(o.name == 'addr' for o in inst.operands)
):
continue
opnd_size_expr = self._operand_size_override(
enc.enc_name, opnd, inst_sem
)
Expand Down Expand Up @@ -9831,7 +9870,7 @@ def gen_insts(self) -> None:
_needs_atomic_return_view = (
_is_optional_atomic_return
and inst_sem.semantic_class == 'buffer_atomic'
and inst_sem.operation == 'cmpswap'
and inst_sem.operation in ('cmpswap', 'fcmpswap')
and opnd.name == 'vdata'
)
atomic_return_operand = (
Expand Down Expand Up @@ -9982,7 +10021,7 @@ def gen_insts(self) -> None:
_has_flat_saddr = self.isa_spec.profile.mnemonic_rule(
enc.enc_name
).use_flat_mnemonic
if _has_flat_saddr:
if _has_flat_saddr and any(o.name == 'addr' for o in inst.operands):
saddr_null = self._saddr_null_expr(enc.enc_name)
private_members.append(cgen.Statement('Operand saddr'))
opnd_ctor_init.append('saddr(0, OperandType::OPR_SREG, 0)')
Expand Down Expand Up @@ -10315,6 +10354,13 @@ def gen_insts(self) -> None:
'SReg_32_XEXEC destination";'
)

if inst.required_flat_segment is not None:
factory_validation_parts.append(
f'if (reinterpret_cast<const {factory_op_encoding}*>(inst)->seg != '
f'{inst.required_flat_segment}u) [[unlikely]] return emit_error.emit() '
f'<< "{inst.name} requires its GLOBAL segment";'
)

# Flat segment-aware operands: adjust addr width and add
# saddr for SCRATCH (seg==1) and GLOBAL (seg==2) segments.
if rule.use_flat_mnemonic:
Expand Down
43 changes: 39 additions & 4 deletions emulation/rocjitsu/lib/python/amdisa/encoding_translator_codegen.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -727,7 +727,7 @@ def _emit_encode_fn(trans, dst_ns, dst_name):
return lines


def _emit_dispatch(translations, src_name, dst_name):
def _emit_dispatch(translations, src_name, dst_name, src_spec, dst_spec):
val_groups: dict[int, list[EncodingTranslation]] = {}
for t in translations:
val_groups.setdefault(t.src_dt_index, []).append(t)
Expand Down Expand Up @@ -777,8 +777,41 @@ def _emit_dispatch(translations, src_name, dst_name):
if t.src_bit_cnt <= 32
else ('w0, w1' if t.src_bit_cnt == 64 else 'w0, w1, w2')
)
lines.append(f' case kEnc_{cn}:')
lines.append(f' return {enc_fn}({dec_fn}({args}), dst_op);')
swap_compare_data = (
t.src_enc_name in ('ENC_DS', 'ENC_VDS')
and src_spec.profile.ds_compare_store_compare_first
!= dst_spec.profile.ds_compare_store_compare_first
)
if swap_compare_data:
opcodes = sorted(
{
inst.opcode
for inst in src_spec.encoding_map[t.src_enc_name].insts
if inst.name.startswith(('DS_CMPST_', 'DS_CMPSTORE_'))
}
)
fields_type = _fields_struct_name(t.src_enc_name)
lines.append(f' case kEnc_{cn}: {{')
lines.append(f' {fields_type} fields = {dec_fn}({args});')
lines.append(' switch (fields.op) {')
for opcode in opcodes:
lines.append(f' case {opcode}:')
if opcodes:
lines.append(' {')
lines.append(
' const uint32_t comparison = fields.data0;'
)
lines.append(' fields.data0 = fields.data1;')
lines.append(' fields.data1 = comparison;')
lines.append(' break;')
lines.append(' }')
lines.append(' default: break;')
lines.append(' }')
lines.append(f' return {enc_fn}(fields, dst_op);')
lines.append(' }')
else:
lines.append(f' case kEnc_{cn}:')
lines.append(f' return {enc_fn}({dec_fn}({args}), dst_op);')
else:
lines.append(f' case kEnc_{cn}: {{')
lines.append(f' const uint8_t seg = (w0 >> 14) & 0x3;')
Expand Down Expand Up @@ -967,7 +1000,9 @@ def generate_encoding_translators(
pair_lines.extend(_emit_decode_fn(t, src_ns, src_name))
for t in translations:
pair_lines.extend(_emit_encode_fn(t, dst_ns, dst_name))
pair_lines.extend(_emit_dispatch(translations, src_name, dst_name))
pair_lines.extend(
_emit_dispatch(translations, src_name, dst_name, src_spec, dst_spec)
)

pair_lines += [
f'}} // namespace {src_name}_to_{dst_name}',
Expand Down
1 change: 1 addition & 0 deletions emulation/rocjitsu/lib/python/amdisa/gpuisa.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,7 @@ def __init__(
self.name = name
self.opcode = opcode
self.operands = operands
self.required_flat_segment: int | None = None
self.available_encodings = available_encodings
self.source_addition = source_addition
# Populated by an optional ISA-variant manifest after the base XML and
Expand Down
Loading
Loading