From 5d3f0737a633d4b8d512ea7c62b1fd4da6740a33 Mon Sep 17 00:00:00 2001 From: Allyson Cauble-Chantrenne Date: Tue, 3 Nov 2026 11:05:08 -0600 Subject: [PATCH 1/5] [rocjitsu][amdisa] Model `s_buffer_atomic_*` instructions * Very similar to `s_atomic` implementation except for how smem_calculate_address computes the address and an extra bounds check --- emulation/rocjitsu/docs/vm-design.md | 5 +- .../lib/python/amdisa/codegen/_generator.py | 7 +- .../rocjitsu/lib/python/amdisa/semantics.py | 39 +- .../python/amdisa/tests/test_sema_derive.py | 20 +- .../amdisa/tests/test_smem_atomic_contract.py | 15 +- .../isa/arch/amdgpu/generated/cdna1/smem.cpp | 223 +++++-- .../isa/arch/amdgpu/generated/cdna1/smem.h | 2 + .../arch/amdgpu/generated/cdna1/smem_exec.cpp | 564 +++++++++++++++--- .../isa/arch/amdgpu/generated/cdna2/smem.cpp | 223 +++++-- .../isa/arch/amdgpu/generated/cdna2/smem.h | 2 + .../arch/amdgpu/generated/cdna2/smem_exec.cpp | 564 +++++++++++++++--- .../isa/arch/amdgpu/generated/cdna3/smem.cpp | 223 +++++-- .../isa/arch/amdgpu/generated/cdna3/smem.h | 2 + .../arch/amdgpu/generated/cdna3/smem_exec.cpp | 564 +++++++++++++++--- .../isa/arch/amdgpu/generated/cdna4/smem.cpp | 223 +++++-- .../isa/arch/amdgpu/generated/cdna4/smem.h | 2 + .../arch/amdgpu/generated/cdna4/smem_exec.cpp | 564 +++++++++++++++--- .../isa/arch/amdgpu/shared/addr_calc_scalar.h | 11 +- .../src/rocjitsu/vm/amdgpu/mem_state.h | 2 + .../rocjitsu/vm/amdgpu/memory_pipeline.cpp | 9 +- .../rocjitsu/tests/scalar_atomic_test.cpp | 240 +++++--- 21 files changed, 2853 insertions(+), 651 deletions(-) diff --git a/emulation/rocjitsu/docs/vm-design.md b/emulation/rocjitsu/docs/vm-design.md index 1b7c0af1a0a5..de16737b5197 100644 --- a/emulation/rocjitsu/docs/vm-design.md +++ b/emulation/rocjitsu/docs/vm-design.md @@ -569,7 +569,10 @@ coherence epoch makes stale K$ lines refetch. The old value is returned to SDATA when GLC is set. Supported: swap, cmpswap, add, sub, smin/umin, smax/umax, and, or, xor, inc, dec. Compare-swap reads the source and then the compare value from SDATA and returns only the old value, to the low half. -`s_buffer_atomic_*` are not yet implemented. +`s_buffer_atomic_*` support the same operations. Their address is the V# base +plus the offset; stride only scales the bound. An atomic whose element is not +wholly in range performs no access and returns zero when GLC is set. A +misaligned `_x2` atomic faults even when out of range. **Cache management instructions:** diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py index c1f07335100f..63aa1b2d86d3 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py @@ -8378,11 +8378,6 @@ def _gen_smem_store( return '\n'.join(L) def _gen_smem_atomic(self, sem: InstructionSemantics) -> str: - # This emitter computes a global address; buffer forms need V# addressing. - if sem.name.upper().startswith('S_BUFFER_ATOMIC_'): - raise ValueError( - f'{sem.name}: smem_atomic supports only global S_ATOMIC_* addressing' - ) # Semantics has no op-less smem_atomic fallback, so a missing op is a # derivation bug rather than an unimplemented variant. op_enum = self._ATOMIC_OP_ENUM.get(sem.operation) @@ -8417,7 +8412,7 @@ def _gen_smem_atomic(self, sem: InstructionSemantics) -> str: self._append_wait_counter_type(L, sem) L.extend( [ - ' auto address = smem_calculate_address(inst_, wf);', + ' auto address = smem_calculate_address(inst_, wf, d.get());', ' if (!address) return;', ' d->addr = *address;', ' set_data(std::move(d));', diff --git a/emulation/rocjitsu/lib/python/amdisa/semantics.py b/emulation/rocjitsu/lib/python/amdisa/semantics.py index f074f03c88f1..982c8427d1e7 100644 --- a/emulation/rocjitsu/lib/python/amdisa/semantics.py +++ b/emulation/rocjitsu/lib/python/amdisa/semantics.py @@ -1877,8 +1877,10 @@ def _derive_vop3p(name: str) -> InstructionSemantics | None: 'U16': (2, False), } -# S_ATOMIC_* operations the CDNA ISAs define and the scalar pipeline supports. -# Other flat atomic operations stay 'nop' even when the flat tables know them. +# S_ATOMIC_* and S_BUFFER_ATOMIC_* operations the CDNA ISAs define and the +# scalar pipeline supports. +# Other flat atomic operations do not derive smem_atomic, even when the flat +# tables know them. _SMEM_ATOMIC_OPS = frozenset( { 'swap', @@ -1928,24 +1930,21 @@ def _derive_smem(name: str) -> InstructionSemantics | None: if 'DCACHE' in upper or upper in ('S_ATC_PROBE', 'S_ATC_PROBE_BUFFER'): return InstructionSemantics(name, 'nop') - if '_ATOMIC_' in upper: - if upper.startswith('S_ATOMIC_'): - suffix = upper[len('S_ATOMIC_') :] - is_x2 = suffix.endswith('_X2') - if is_x2: - suffix = suffix[:-3] - info = _derive_flat_atomic_info(suffix, is_x2) - if info and info[0] in _SMEM_ATOMIC_OPS: - op, elem_size, data_dw_actual = info - return InstructionSemantics( - name, - 'smem_atomic', - operation=op, - elem_size=elem_size, - num_elems=data_dw_actual, - ) - # Scalar buffer atomics are not currently simulated. - return InstructionSemantics(name, 'nop') + if upper.startswith(('S_ATOMIC_', 'S_BUFFER_ATOMIC_')): + suffix = upper.partition('_ATOMIC_')[2] + is_x2 = suffix.endswith('_X2') + if is_x2: + suffix = suffix[:-3] + info = _derive_flat_atomic_info(suffix, is_x2) + if info and info[0] in _SMEM_ATOMIC_OPS: + op, elem_size, data_dw_actual = info + return InstructionSemantics( + name, + 'smem_atomic', + operation=op, + elem_size=elem_size, + num_elems=data_dw_actual, + ) is_store = '_STORE_' in upper or '_SCRATCH_STORE_' in upper if not is_store: diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py index 109357baaba9..4f7a95a1bff9 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py @@ -2377,11 +2377,12 @@ def test_gfx1250_prefetch_hints_are_true_nop(self, name): # Compare-swap carries its compare value after the source, doubling its data. _IMPLEMENTED_SCALAR_ATOMICS = { - f'S_ATOMIC_{op}{width}': ( + f'{prefix}{op}{width}': ( op.lower(), elem_size, 2 * num_elems if op == 'CMPSWAP' else num_elems, ) + for prefix in ('S_ATOMIC_', 'S_BUFFER_ATOMIC_') for op in _SCALAR_ATOMIC_OPS for width, elem_size, num_elems in (('', 4, 1), ('_X2', 8, 2)) } @@ -2398,21 +2399,6 @@ def test_implemented_scalar_atomics_derive_operation_and_widths(self, name): assert sem.elem_size == elem_size assert sem.num_elems == num_elems - @pytest.mark.parametrize( - 'name', - [ - f'{prefix}{op}{width}' - for prefix in ('S_ATOMIC_', 'S_BUFFER_ATOMIC_') - for op in _SCALAR_ATOMIC_OPS - for width in ('', '_X2') - if f'{prefix}{op}{width}' not in _IMPLEMENTED_SCALAR_ATOMICS - ], - ) - def test_unimplemented_scalar_atomics_stay_nop(self, name): - sem = derive_semantics(name, 'ENC_SMEM') - assert sem is not None - assert sem.semantic_class == 'nop' - # No CDNA ISA defines these. The flat atomic tables know their operations, # so they pin that scalar support comes from the allowlist, not that table. @pytest.mark.parametrize( @@ -2423,6 +2409,8 @@ def test_unimplemented_scalar_atomics_stay_nop(self, name): 'S_ATOMIC_PK_ADD_F16', 'S_ATOMIC_CSUB', 'S_ATOMIC_COND_SUB_U32', + 'S_BUFFER_ATOMIC_ADD_F32', + 'S_BUFFER_ATOMIC_FMIN', ], ) def test_scalar_atomic_operations_outside_allowlist_stay_nop(self, name): diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_smem_atomic_contract.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_smem_atomic_contract.py index 5b353eb9ecb2..e34673101079 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_smem_atomic_contract.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_smem_atomic_contract.py @@ -31,8 +31,11 @@ def _codegen(arch_name, profile): ('cdna4', Cdna4Profile()), ], ) -@pytest.mark.parametrize('name', ['S_ATOMIC_DEC', 's_atomic_dec']) -def test_smem_atomic_accepts_derived_global_decrement(arch_name, profile, name): +@pytest.mark.parametrize( + 'name', + ['S_ATOMIC_DEC', 's_atomic_dec', 'S_BUFFER_ATOMIC_DEC', 's_buffer_atomic_dec'], +) +def test_smem_atomic_accepts_derived_decrement(arch_name, profile, name): sem = derive_semantics(name, 'ENC_SMEM') assert sem is not None assert sem.semantic_class == 'smem_atomic' @@ -42,7 +45,8 @@ def test_smem_atomic_accepts_derived_global_decrement(arch_name, profile, name): assert 'd->atomic_op = amdgpu::AtomicOp::DEC;' in body assert 'd->elem_size = 4;' in body assert 'd->num_dwords = 1;' in body - assert 'smem_calculate_address(inst_, wf);' in body + # The state receives the buffer bounds mask; global atomics leave it full. + assert 'smem_calculate_address(inst_, wf, d.get());' in body @pytest.mark.parametrize( @@ -85,11 +89,6 @@ def test_smem_atomic_payload_and_return_widths_follow_semantics( @pytest.mark.parametrize( ('changes', 'message'), [ - pytest.param( - {'name': 'S_BUFFER_ATOMIC_DEC'}, - 'supports only global S_ATOMIC_', - id='buffer-mnemonic', - ), pytest.param( {'operation': 'unknown'}, 'unsupported smem_atomic operation', diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/smem.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/smem.cpp index 61555bcac545..53cdf676f4ed 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/smem.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/smem.cpp @@ -955,11 +955,15 @@ SBufferAtomicSwapSmem::SBufferAtomicSwapSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -978,14 +982,19 @@ SBufferAtomicCmpswapSmem::SBufferAtomicCmpswapSmem(const MachineInst *inst) : Smem("s_buffer_atomic_cmpswap", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::SBufferAtomicCmpswapSmem)), sdata(64, OperandType::OPR_SREG, reinterpret_cast(inst)->sdata), + sdata_return(32, OperandType::OPR_SREG, reinterpret_cast(inst)->sdata), sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata_return; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1007,11 +1016,15 @@ SBufferAtomicAddSmem::SBufferAtomicAddSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1033,11 +1046,15 @@ SBufferAtomicSubSmem::SBufferAtomicSubSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1059,11 +1076,15 @@ SBufferAtomicSminSmem::SBufferAtomicSminSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1085,11 +1106,15 @@ SBufferAtomicUminSmem::SBufferAtomicUminSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1111,11 +1136,15 @@ SBufferAtomicSmaxSmem::SBufferAtomicSmaxSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1137,11 +1166,15 @@ SBufferAtomicUmaxSmem::SBufferAtomicUmaxSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1163,11 +1196,15 @@ SBufferAtomicAndSmem::SBufferAtomicAndSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1189,11 +1226,15 @@ SBufferAtomicOrSmem::SBufferAtomicOrSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1215,11 +1256,15 @@ SBufferAtomicXorSmem::SBufferAtomicXorSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1241,11 +1286,15 @@ SBufferAtomicIncSmem::SBufferAtomicIncSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1267,11 +1316,15 @@ SBufferAtomicDecSmem::SBufferAtomicDecSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1293,11 +1346,16 @@ SBufferAtomicSwapX2Smem::SBufferAtomicSwapX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1316,14 +1374,20 @@ SBufferAtomicCmpswapX2Smem::SBufferAtomicCmpswapX2Smem(const MachineInst *inst) : Smem("s_buffer_atomic_cmpswap_x2", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::SBufferAtomicCmpswapX2Smem)), sdata(128, OperandType::OPR_SREG, reinterpret_cast(inst)->sdata), + sdata_return(64, OperandType::OPR_SREG, reinterpret_cast(inst)->sdata), sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata_return; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1345,11 +1409,16 @@ SBufferAtomicAddX2Smem::SBufferAtomicAddX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1371,11 +1440,16 @@ SBufferAtomicSubX2Smem::SBufferAtomicSubX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1397,11 +1471,16 @@ SBufferAtomicSminX2Smem::SBufferAtomicSminX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1423,11 +1502,16 @@ SBufferAtomicUminX2Smem::SBufferAtomicUminX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1449,11 +1533,16 @@ SBufferAtomicSmaxX2Smem::SBufferAtomicSmaxX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1475,11 +1564,16 @@ SBufferAtomicUmaxX2Smem::SBufferAtomicUmaxX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1501,11 +1595,16 @@ SBufferAtomicAndX2Smem::SBufferAtomicAndX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1527,11 +1626,16 @@ SBufferAtomicOrX2Smem::SBufferAtomicOrX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1553,11 +1657,16 @@ SBufferAtomicXorX2Smem::SBufferAtomicXorX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1579,11 +1688,16 @@ SBufferAtomicIncX2Smem::SBufferAtomicIncX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1605,11 +1719,16 @@ SBufferAtomicDecX2Smem::SBufferAtomicDecX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/smem.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/smem.h index 50697eaab7cb..30fffc147d13 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/smem.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/smem.h @@ -352,6 +352,7 @@ class SBufferAtomicCmpswapSmem : public Smem { SBufferAtomicCmpswapSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); Operand sdata; + Operand sdata_return; Operand sbase; Operand soffset; }; @@ -469,6 +470,7 @@ class SBufferAtomicCmpswapX2Smem : public Smem { SBufferAtomicCmpswapX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); Operand sdata; + Operand sdata_return; Operand sbase; Operand soffset; }; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/smem_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/smem_exec.cpp index c816f6f3a978..4e21f9611009 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/smem_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/smem_exec.cpp @@ -484,133 +484,541 @@ void SDcacheDiscardX2Smem::execute_impl(amdgpu::Wavefront &wf) { } void SBufferAtomicSwapSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::SWAP; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicCmpswapSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->dst_register.width = 1; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::CMPSWAP; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicAddSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::ADD; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSubSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::SUB; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSminSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::SMIN; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicUminSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::UMIN; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSmaxSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::SMAX; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicUmaxSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::UMAX; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicAndSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::AND; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicOrSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::OR; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicXorSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::XOR; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicIncSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::INC; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicDecSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::DEC; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSwapX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::SWAP; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicCmpswapX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 4u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->dst_register.width = 2; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::CMPSWAP; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->store_data[2] = amdgpu::read_scalar_register(wf, *data_register, 2); + d->store_data[3] = amdgpu::read_scalar_register(wf, *data_register, 3); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicAddX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::ADD; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSubX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::SUB; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSminX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::SMIN; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicUminX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::UMIN; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSmaxX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::SMAX; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicUmaxX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::UMAX; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicAndX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::AND; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicOrX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::OR; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicXorX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::XOR; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicIncX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::INC; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicDecX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::DEC; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SAtomicSwapSmem::execute_impl(amdgpu::Wavefront &wf) { @@ -626,7 +1034,7 @@ void SAtomicSwapSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -648,7 +1056,7 @@ void SAtomicCmpswapSmem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -668,7 +1076,7 @@ void SAtomicAddSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -688,7 +1096,7 @@ void SAtomicSubSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -708,7 +1116,7 @@ void SAtomicSminSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -728,7 +1136,7 @@ void SAtomicUminSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -748,7 +1156,7 @@ void SAtomicSmaxSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -768,7 +1176,7 @@ void SAtomicUmaxSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -788,7 +1196,7 @@ void SAtomicAndSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -808,7 +1216,7 @@ void SAtomicOrSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -828,7 +1236,7 @@ void SAtomicXorSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -848,7 +1256,7 @@ void SAtomicIncSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -868,7 +1276,7 @@ void SAtomicDecSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -889,7 +1297,7 @@ void SAtomicSwapX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -913,7 +1321,7 @@ void SAtomicCmpswapX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[2] = amdgpu::read_scalar_register(wf, *data_register, 2); d->store_data[3] = amdgpu::read_scalar_register(wf, *data_register, 3); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -934,7 +1342,7 @@ void SAtomicAddX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -955,7 +1363,7 @@ void SAtomicSubX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -976,7 +1384,7 @@ void SAtomicSminX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -997,7 +1405,7 @@ void SAtomicUminX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1018,7 +1426,7 @@ void SAtomicSmaxX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1039,7 +1447,7 @@ void SAtomicUmaxX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1060,7 +1468,7 @@ void SAtomicAndX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1081,7 +1489,7 @@ void SAtomicOrX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1102,7 +1510,7 @@ void SAtomicXorX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1123,7 +1531,7 @@ void SAtomicIncX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1144,7 +1552,7 @@ void SAtomicDecX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/smem.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/smem.cpp index 229ca8de53f2..0f648d55ed51 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/smem.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/smem.cpp @@ -955,11 +955,15 @@ SBufferAtomicSwapSmem::SBufferAtomicSwapSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -978,14 +982,19 @@ SBufferAtomicCmpswapSmem::SBufferAtomicCmpswapSmem(const MachineInst *inst) : Smem("s_buffer_atomic_cmpswap", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::SBufferAtomicCmpswapSmem)), sdata(64, OperandType::OPR_SREG, reinterpret_cast(inst)->sdata), + sdata_return(32, OperandType::OPR_SREG, reinterpret_cast(inst)->sdata), sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata_return; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1007,11 +1016,15 @@ SBufferAtomicAddSmem::SBufferAtomicAddSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1033,11 +1046,15 @@ SBufferAtomicSubSmem::SBufferAtomicSubSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1059,11 +1076,15 @@ SBufferAtomicSminSmem::SBufferAtomicSminSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1085,11 +1106,15 @@ SBufferAtomicUminSmem::SBufferAtomicUminSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1111,11 +1136,15 @@ SBufferAtomicSmaxSmem::SBufferAtomicSmaxSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1137,11 +1166,15 @@ SBufferAtomicUmaxSmem::SBufferAtomicUmaxSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1163,11 +1196,15 @@ SBufferAtomicAndSmem::SBufferAtomicAndSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1189,11 +1226,15 @@ SBufferAtomicOrSmem::SBufferAtomicOrSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1215,11 +1256,15 @@ SBufferAtomicXorSmem::SBufferAtomicXorSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1241,11 +1286,15 @@ SBufferAtomicIncSmem::SBufferAtomicIncSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1267,11 +1316,15 @@ SBufferAtomicDecSmem::SBufferAtomicDecSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1293,11 +1346,16 @@ SBufferAtomicSwapX2Smem::SBufferAtomicSwapX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1316,14 +1374,20 @@ SBufferAtomicCmpswapX2Smem::SBufferAtomicCmpswapX2Smem(const MachineInst *inst) : Smem("s_buffer_atomic_cmpswap_x2", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::SBufferAtomicCmpswapX2Smem)), sdata(128, OperandType::OPR_SREG, reinterpret_cast(inst)->sdata), + sdata_return(64, OperandType::OPR_SREG, reinterpret_cast(inst)->sdata), sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata_return; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1345,11 +1409,16 @@ SBufferAtomicAddX2Smem::SBufferAtomicAddX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1371,11 +1440,16 @@ SBufferAtomicSubX2Smem::SBufferAtomicSubX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1397,11 +1471,16 @@ SBufferAtomicSminX2Smem::SBufferAtomicSminX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1423,11 +1502,16 @@ SBufferAtomicUminX2Smem::SBufferAtomicUminX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1449,11 +1533,16 @@ SBufferAtomicSmaxX2Smem::SBufferAtomicSmaxX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1475,11 +1564,16 @@ SBufferAtomicUmaxX2Smem::SBufferAtomicUmaxX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1501,11 +1595,16 @@ SBufferAtomicAndX2Smem::SBufferAtomicAndX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1527,11 +1626,16 @@ SBufferAtomicOrX2Smem::SBufferAtomicOrX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1553,11 +1657,16 @@ SBufferAtomicXorX2Smem::SBufferAtomicXorX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1579,11 +1688,16 @@ SBufferAtomicIncX2Smem::SBufferAtomicIncX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1605,11 +1719,16 @@ SBufferAtomicDecX2Smem::SBufferAtomicDecX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/smem.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/smem.h index c1d5304ed74c..4ed82e7b7f19 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/smem.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/smem.h @@ -352,6 +352,7 @@ class SBufferAtomicCmpswapSmem : public Smem { SBufferAtomicCmpswapSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); Operand sdata; + Operand sdata_return; Operand sbase; Operand soffset; }; @@ -469,6 +470,7 @@ class SBufferAtomicCmpswapX2Smem : public Smem { SBufferAtomicCmpswapX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); Operand sdata; + Operand sdata_return; Operand sbase; Operand soffset; }; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/smem_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/smem_exec.cpp index 708d8bdfb81c..bbd30d612b98 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/smem_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/smem_exec.cpp @@ -484,133 +484,541 @@ void SDcacheDiscardX2Smem::execute_impl(amdgpu::Wavefront &wf) { } void SBufferAtomicSwapSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::SWAP; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicCmpswapSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->dst_register.width = 1; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::CMPSWAP; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicAddSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::ADD; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSubSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::SUB; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSminSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::SMIN; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicUminSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::UMIN; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSmaxSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::SMAX; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicUmaxSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::UMAX; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicAndSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::AND; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicOrSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::OR; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicXorSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::XOR; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicIncSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::INC; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicDecSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::DEC; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSwapX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::SWAP; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicCmpswapX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 4u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->dst_register.width = 2; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::CMPSWAP; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->store_data[2] = amdgpu::read_scalar_register(wf, *data_register, 2); + d->store_data[3] = amdgpu::read_scalar_register(wf, *data_register, 3); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicAddX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::ADD; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSubX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::SUB; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSminX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::SMIN; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicUminX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::UMIN; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSmaxX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::SMAX; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicUmaxX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::UMAX; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicAndX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::AND; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicOrX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::OR; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicXorX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::XOR; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicIncX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::INC; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicDecX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::DEC; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SAtomicSwapSmem::execute_impl(amdgpu::Wavefront &wf) { @@ -626,7 +1034,7 @@ void SAtomicSwapSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -648,7 +1056,7 @@ void SAtomicCmpswapSmem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -668,7 +1076,7 @@ void SAtomicAddSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -688,7 +1096,7 @@ void SAtomicSubSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -708,7 +1116,7 @@ void SAtomicSminSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -728,7 +1136,7 @@ void SAtomicUminSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -748,7 +1156,7 @@ void SAtomicSmaxSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -768,7 +1176,7 @@ void SAtomicUmaxSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -788,7 +1196,7 @@ void SAtomicAndSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -808,7 +1216,7 @@ void SAtomicOrSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -828,7 +1236,7 @@ void SAtomicXorSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -848,7 +1256,7 @@ void SAtomicIncSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -868,7 +1276,7 @@ void SAtomicDecSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -889,7 +1297,7 @@ void SAtomicSwapX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -913,7 +1321,7 @@ void SAtomicCmpswapX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[2] = amdgpu::read_scalar_register(wf, *data_register, 2); d->store_data[3] = amdgpu::read_scalar_register(wf, *data_register, 3); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -934,7 +1342,7 @@ void SAtomicAddX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -955,7 +1363,7 @@ void SAtomicSubX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -976,7 +1384,7 @@ void SAtomicSminX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -997,7 +1405,7 @@ void SAtomicUminX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1018,7 +1426,7 @@ void SAtomicSmaxX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1039,7 +1447,7 @@ void SAtomicUmaxX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1060,7 +1468,7 @@ void SAtomicAndX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1081,7 +1489,7 @@ void SAtomicOrX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1102,7 +1510,7 @@ void SAtomicXorX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1123,7 +1531,7 @@ void SAtomicIncX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1144,7 +1552,7 @@ void SAtomicDecX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/smem.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/smem.cpp index 12ed3f3efa69..5a61617227c8 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/smem.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/smem.cpp @@ -956,11 +956,15 @@ SBufferAtomicSwapSmem::SBufferAtomicSwapSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -979,14 +983,19 @@ SBufferAtomicCmpswapSmem::SBufferAtomicCmpswapSmem(const MachineInst *inst) : Smem("s_buffer_atomic_cmpswap", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::SBufferAtomicCmpswapSmem)), sdata(64, OperandType::OPR_SREG, reinterpret_cast(inst)->sdata), + sdata_return(32, OperandType::OPR_SREG, reinterpret_cast(inst)->sdata), sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata_return; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1008,11 +1017,15 @@ SBufferAtomicAddSmem::SBufferAtomicAddSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1034,11 +1047,15 @@ SBufferAtomicSubSmem::SBufferAtomicSubSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1060,11 +1077,15 @@ SBufferAtomicSminSmem::SBufferAtomicSminSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1086,11 +1107,15 @@ SBufferAtomicUminSmem::SBufferAtomicUminSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1112,11 +1137,15 @@ SBufferAtomicSmaxSmem::SBufferAtomicSmaxSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1138,11 +1167,15 @@ SBufferAtomicUmaxSmem::SBufferAtomicUmaxSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1164,11 +1197,15 @@ SBufferAtomicAndSmem::SBufferAtomicAndSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1190,11 +1227,15 @@ SBufferAtomicOrSmem::SBufferAtomicOrSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1216,11 +1257,15 @@ SBufferAtomicXorSmem::SBufferAtomicXorSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1242,11 +1287,15 @@ SBufferAtomicIncSmem::SBufferAtomicIncSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1268,11 +1317,15 @@ SBufferAtomicDecSmem::SBufferAtomicDecSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1294,11 +1347,16 @@ SBufferAtomicSwapX2Smem::SBufferAtomicSwapX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1317,14 +1375,20 @@ SBufferAtomicCmpswapX2Smem::SBufferAtomicCmpswapX2Smem(const MachineInst *inst) : Smem("s_buffer_atomic_cmpswap_x2", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::SBufferAtomicCmpswapX2Smem)), sdata(128, OperandType::OPR_SREG, reinterpret_cast(inst)->sdata), + sdata_return(64, OperandType::OPR_SREG, reinterpret_cast(inst)->sdata), sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata_return; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1346,11 +1410,16 @@ SBufferAtomicAddX2Smem::SBufferAtomicAddX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1372,11 +1441,16 @@ SBufferAtomicSubX2Smem::SBufferAtomicSubX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1398,11 +1472,16 @@ SBufferAtomicSminX2Smem::SBufferAtomicSminX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1424,11 +1503,16 @@ SBufferAtomicUminX2Smem::SBufferAtomicUminX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1450,11 +1534,16 @@ SBufferAtomicSmaxX2Smem::SBufferAtomicSmaxX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1476,11 +1565,16 @@ SBufferAtomicUmaxX2Smem::SBufferAtomicUmaxX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1502,11 +1596,16 @@ SBufferAtomicAndX2Smem::SBufferAtomicAndX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1528,11 +1627,16 @@ SBufferAtomicOrX2Smem::SBufferAtomicOrX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1554,11 +1658,16 @@ SBufferAtomicXorX2Smem::SBufferAtomicXorX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1580,11 +1689,16 @@ SBufferAtomicIncX2Smem::SBufferAtomicIncX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1606,11 +1720,16 @@ SBufferAtomicDecX2Smem::SBufferAtomicDecX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/smem.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/smem.h index 059aac42af4e..c2e8c72fabce 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/smem.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/smem.h @@ -352,6 +352,7 @@ class SBufferAtomicCmpswapSmem : public Smem { SBufferAtomicCmpswapSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); Operand sdata; + Operand sdata_return; Operand sbase; Operand soffset; }; @@ -469,6 +470,7 @@ class SBufferAtomicCmpswapX2Smem : public Smem { SBufferAtomicCmpswapX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); Operand sdata; + Operand sdata_return; Operand sbase; Operand soffset; }; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/smem_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/smem_exec.cpp index 94cf01c92bb0..c230b58fe48a 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/smem_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/smem_exec.cpp @@ -485,133 +485,541 @@ void SDcacheDiscardX2Smem::execute_impl(amdgpu::Wavefront &wf) { } void SBufferAtomicSwapSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::SWAP; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicCmpswapSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->dst_register.width = 1; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::CMPSWAP; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicAddSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::ADD; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSubSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::SUB; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSminSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::SMIN; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicUminSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::UMIN; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSmaxSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::SMAX; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicUmaxSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::UMAX; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicAndSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::AND; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicOrSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::OR; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicXorSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::XOR; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicIncSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::INC; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicDecSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::DEC; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSwapX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::SWAP; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicCmpswapX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 4u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->dst_register.width = 2; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::CMPSWAP; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->store_data[2] = amdgpu::read_scalar_register(wf, *data_register, 2); + d->store_data[3] = amdgpu::read_scalar_register(wf, *data_register, 3); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicAddX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::ADD; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSubX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::SUB; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSminX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::SMIN; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicUminX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::UMIN; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSmaxX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::SMAX; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicUmaxX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::UMAX; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicAndX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::AND; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicOrX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::OR; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicXorX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::XOR; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicIncX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::INC; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicDecX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::DEC; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SAtomicSwapSmem::execute_impl(amdgpu::Wavefront &wf) { @@ -627,7 +1035,7 @@ void SAtomicSwapSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -649,7 +1057,7 @@ void SAtomicCmpswapSmem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -669,7 +1077,7 @@ void SAtomicAddSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -689,7 +1097,7 @@ void SAtomicSubSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -709,7 +1117,7 @@ void SAtomicSminSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -729,7 +1137,7 @@ void SAtomicUminSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -749,7 +1157,7 @@ void SAtomicSmaxSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -769,7 +1177,7 @@ void SAtomicUmaxSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -789,7 +1197,7 @@ void SAtomicAndSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -809,7 +1217,7 @@ void SAtomicOrSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -829,7 +1237,7 @@ void SAtomicXorSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -849,7 +1257,7 @@ void SAtomicIncSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -869,7 +1277,7 @@ void SAtomicDecSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -890,7 +1298,7 @@ void SAtomicSwapX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -914,7 +1322,7 @@ void SAtomicCmpswapX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[2] = amdgpu::read_scalar_register(wf, *data_register, 2); d->store_data[3] = amdgpu::read_scalar_register(wf, *data_register, 3); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -935,7 +1343,7 @@ void SAtomicAddX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -956,7 +1364,7 @@ void SAtomicSubX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -977,7 +1385,7 @@ void SAtomicSminX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -998,7 +1406,7 @@ void SAtomicUminX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1019,7 +1427,7 @@ void SAtomicSmaxX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1040,7 +1448,7 @@ void SAtomicUmaxX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1061,7 +1469,7 @@ void SAtomicAndX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1082,7 +1490,7 @@ void SAtomicOrX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1103,7 +1511,7 @@ void SAtomicXorX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1124,7 +1532,7 @@ void SAtomicIncX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1145,7 +1553,7 @@ void SAtomicDecX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/smem.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/smem.cpp index 3de348719808..58a6eb9e75e4 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/smem.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/smem.cpp @@ -956,11 +956,15 @@ SBufferAtomicSwapSmem::SBufferAtomicSwapSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -979,14 +983,19 @@ SBufferAtomicCmpswapSmem::SBufferAtomicCmpswapSmem(const MachineInst *inst) : Smem("s_buffer_atomic_cmpswap", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::SBufferAtomicCmpswapSmem)), sdata(64, OperandType::OPR_SREG, reinterpret_cast(inst)->sdata), + sdata_return(32, OperandType::OPR_SREG, reinterpret_cast(inst)->sdata), sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata_return; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1008,11 +1017,15 @@ SBufferAtomicAddSmem::SBufferAtomicAddSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1034,11 +1047,15 @@ SBufferAtomicSubSmem::SBufferAtomicSubSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1060,11 +1077,15 @@ SBufferAtomicSminSmem::SBufferAtomicSminSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1086,11 +1107,15 @@ SBufferAtomicUminSmem::SBufferAtomicUminSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1112,11 +1137,15 @@ SBufferAtomicSmaxSmem::SBufferAtomicSmaxSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1138,11 +1167,15 @@ SBufferAtomicUmaxSmem::SBufferAtomicUmaxSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1164,11 +1197,15 @@ SBufferAtomicAndSmem::SBufferAtomicAndSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1190,11 +1227,15 @@ SBufferAtomicOrSmem::SBufferAtomicOrSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1216,11 +1257,15 @@ SBufferAtomicXorSmem::SBufferAtomicXorSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1242,11 +1287,15 @@ SBufferAtomicIncSmem::SBufferAtomicIncSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1268,11 +1317,15 @@ SBufferAtomicDecSmem::SBufferAtomicDecSmem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info({amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1294,11 +1347,16 @@ SBufferAtomicSwapX2Smem::SBufferAtomicSwapX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1317,14 +1375,20 @@ SBufferAtomicCmpswapX2Smem::SBufferAtomicCmpswapX2Smem(const MachineInst *inst) : Smem("s_buffer_atomic_cmpswap_x2", reinterpret_cast(inst), selected_exec_fn(InstructionExecutionId::SBufferAtomicCmpswapX2Smem)), sdata(128, OperandType::OPR_SREG, reinterpret_cast(inst)->sdata), + sdata_return(64, OperandType::OPR_SREG, reinterpret_cast(inst)->sdata), sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata_return; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1346,11 +1410,16 @@ SBufferAtomicAddX2Smem::SBufferAtomicAddX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1372,11 +1441,16 @@ SBufferAtomicSubX2Smem::SBufferAtomicSubX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1398,11 +1472,16 @@ SBufferAtomicSminX2Smem::SBufferAtomicSminX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1424,11 +1503,16 @@ SBufferAtomicUminX2Smem::SBufferAtomicUminX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1450,11 +1534,16 @@ SBufferAtomicSmaxX2Smem::SBufferAtomicSmaxX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1476,11 +1565,16 @@ SBufferAtomicUmaxX2Smem::SBufferAtomicUmaxX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1502,11 +1596,16 @@ SBufferAtomicAndX2Smem::SBufferAtomicAndX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1528,11 +1627,16 @@ SBufferAtomicOrX2Smem::SBufferAtomicOrX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1554,11 +1658,16 @@ SBufferAtomicXorX2Smem::SBufferAtomicXorX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1580,11 +1689,16 @@ SBufferAtomicIncX2Smem::SBufferAtomicIncX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } @@ -1606,11 +1720,16 @@ SBufferAtomicDecX2Smem::SBufferAtomicDecX2Smem(const MachineInst *inst) sbase(128, OperandType::OPR_SREG, (reinterpret_cast(inst)->sbase * 2)), soffset(make_smem_offset(reinterpret_cast(inst))) { src_operands_[0] = &sdata; - dst_operands_[0] = &sdata; src_operands_[1] = &sbase; src_operands_[2] = &soffset; num_src_ = 3; - num_dst_ = 1; + num_dst_ = 0; + if (inst_.glc != 0) + dst_operands_[num_dst_++] = &sdata; + set_memory_issue_info( + {amdgpu::MemoryCounterObligation{amdgpu::WaitCounterType::LGKMCNT, + amdgpu::MemoryCompletionClass::UNORDERED, 2}}, + false); flags_ |= MEMORY_WAIT_PRODUCER; } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/smem.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/smem.h index 7e2651714c4b..e5fa032f1d24 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/smem.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/smem.h @@ -352,6 +352,7 @@ class SBufferAtomicCmpswapSmem : public Smem { SBufferAtomicCmpswapSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); Operand sdata; + Operand sdata_return; Operand sbase; Operand soffset; }; @@ -469,6 +470,7 @@ class SBufferAtomicCmpswapX2Smem : public Smem { SBufferAtomicCmpswapX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); Operand sdata; + Operand sdata_return; Operand sbase; Operand soffset; }; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/smem_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/smem_exec.cpp index b2e28bab06ec..52997342b904 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/smem_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/smem_exec.cpp @@ -485,133 +485,541 @@ void SDcacheDiscardX2Smem::execute_impl(amdgpu::Wavefront &wf) { } void SBufferAtomicSwapSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::SWAP; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicCmpswapSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->dst_register.width = 1; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::CMPSWAP; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicAddSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::ADD; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSubSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::SUB; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSminSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::SMIN; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicUminSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::UMIN; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSmaxSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::SMAX; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicUmaxSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::UMAX; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicAndSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::AND; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicOrSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::OR; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicXorSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::XOR; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicIncSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::INC; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicDecSmem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 1u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 1; + d->elem_size = 4; + d->atomic_op = amdgpu::AtomicOp::DEC; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSwapX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::SWAP; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicCmpswapX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 4u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->dst_register.width = 2; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::CMPSWAP; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->store_data[2] = amdgpu::read_scalar_register(wf, *data_register, 2); + d->store_data[3] = amdgpu::read_scalar_register(wf, *data_register, 3); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicAddX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::ADD; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSubX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::SUB; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSminX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::SMIN; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicUminX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::UMIN; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicSmaxX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::SMAX; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicUmaxX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::UMAX; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicAndX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::AND; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicOrX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::OR; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicXorX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::XOR; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicIncX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::INC; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SBufferAtomicDecX2Smem::execute_impl(amdgpu::Wavefront &wf) { - wf.report_instruction_execution_error( - amdgpu::InstructionExecutionError::UnimplementedInstruction); + auto data_register = amdgpu::resolve_scalar_register_range(wf, inst_.sdata, 2u); + if (!data_register) + return; + auto d = std::make_unique(); + d->dst_register = *data_register; + d->num_dwords = 2; + d->elem_size = 8; + d->atomic_op = amdgpu::AtomicOp::DEC; + d->is_load = inst_.glc != 0; + d->mtype = amdgpu::Mtype::UC; + d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); + d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); + d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; + auto address = smem_calculate_address(inst_, wf, d.get()); + if (!address) + return; + d->addr = *address; + set_data(std::move(d)); } void SAtomicSwapSmem::execute_impl(amdgpu::Wavefront &wf) { @@ -627,7 +1035,7 @@ void SAtomicSwapSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -649,7 +1057,7 @@ void SAtomicCmpswapSmem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -669,7 +1077,7 @@ void SAtomicAddSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -689,7 +1097,7 @@ void SAtomicSubSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -709,7 +1117,7 @@ void SAtomicSminSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -729,7 +1137,7 @@ void SAtomicUminSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -749,7 +1157,7 @@ void SAtomicSmaxSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -769,7 +1177,7 @@ void SAtomicUmaxSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -789,7 +1197,7 @@ void SAtomicAndSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -809,7 +1217,7 @@ void SAtomicOrSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -829,7 +1237,7 @@ void SAtomicXorSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -849,7 +1257,7 @@ void SAtomicIncSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -869,7 +1277,7 @@ void SAtomicDecSmem::execute_impl(amdgpu::Wavefront &wf) { d->mtype = amdgpu::Mtype::UC; d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -890,7 +1298,7 @@ void SAtomicSwapX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -914,7 +1322,7 @@ void SAtomicCmpswapX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[2] = amdgpu::read_scalar_register(wf, *data_register, 2); d->store_data[3] = amdgpu::read_scalar_register(wf, *data_register, 3); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -935,7 +1343,7 @@ void SAtomicAddX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -956,7 +1364,7 @@ void SAtomicSubX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -977,7 +1385,7 @@ void SAtomicSminX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -998,7 +1406,7 @@ void SAtomicUminX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1019,7 +1427,7 @@ void SAtomicSmaxX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1040,7 +1448,7 @@ void SAtomicUmaxX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1061,7 +1469,7 @@ void SAtomicAndX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1082,7 +1490,7 @@ void SAtomicOrX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1103,7 +1511,7 @@ void SAtomicXorX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1124,7 +1532,7 @@ void SAtomicIncX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; @@ -1145,7 +1553,7 @@ void SAtomicDecX2Smem::execute_impl(amdgpu::Wavefront &wf) { d->store_data[0] = amdgpu::read_scalar_register(wf, *data_register, 0); d->store_data[1] = amdgpu::read_scalar_register(wf, *data_register, 1); d->wait_counter_type = amdgpu::WaitCounterType::LGKMCNT; - auto address = smem_calculate_address(inst_, wf); + auto address = smem_calculate_address(inst_, wf, d.get()); if (!address) return; d->addr = *address; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/addr_calc_scalar.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/addr_calc_scalar.h index e0e95cd383ee..909e3593d995 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/addr_calc_scalar.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/addr_calc_scalar.h @@ -59,7 +59,16 @@ inline std::optional scalar_buffer_address(Wavefront &wf, uint32_t sel return std::nullopt; const uint64_t stride = (base >> 48) & 0x3fff; scalar_buffer_load_mask(state, offset, (*records * (stride ? stride : 1)) & ~size_align_mask); - return buffer_virtual_address(base + offset) & ~align_mask; + const uint64_t address = buffer_virtual_address(base + offset) & ~align_mask; + if (state && state->atomic_op != AtomicOp::NONE) { + // An atomic accesses its whole element or nothing. Hardware checks atomic + // alignment before bounds, so a misaligned atomic keeps its full mask and + // faults in the access path. + const uint16_t element_mask = (1u << state->num_dwords) - 1; + const bool aligned = address % state->elem_size == 0; + state->load_dword_mask = state->load_dword_mask == element_mask || !aligned ? element_mask : 0; + } + return address; } /// @brief Scalar-buffer load opcode families (GFX9/RDNA through GFX11, then GFX12). diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/mem_state.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/mem_state.h index 9f0133903152..b4b04c927b00 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/mem_state.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/mem_state.h @@ -95,6 +95,8 @@ class ScalarMemState : public DynamicInstState { AtomicOp atomic_op = AtomicOp::NONE; Mtype mtype = Mtype::RW; WaitCounterType wait_counter_type = WaitCounterType::LGKMCNT; + /// In-range dwords of a buffer access: masked load dwords read zero and masked + /// store dwords are skipped. An atomic's mask covers its whole element or is zero. uint16_t load_dword_mask = 0xffff; uint32_t response_data[16] = {}; uint32_t store_data[16] = {}; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/memory_pipeline.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/memory_pipeline.cpp index 42413d17db7a..37f4ef6bc832 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/memory_pipeline.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/memory_pipeline.cpp @@ -607,11 +607,14 @@ VmAccessOutcome ScalarMemPipeline::initiate_access(Instruction &inst, Wavefront ((d.elem_size != sizeof(uint32_t) && d.elem_size != sizeof(uint64_t)) || d.num_dwords * sizeof(uint32_t) != d.elem_size)) return VmAccessOutcome::Malformed; - // Masked loads still write the zero response, without taking a VM snapshot or - // touching backing memory, including byte and halfword requests. - // Fully masked stores likewise perform no VM or backing-memory access. + // Masked loads and returning atomics still write the zero response, without + // taking a VM snapshot or touching backing memory, including byte and halfword + // requests. Fully masked stores likewise perform no VM or backing-memory access. if (d.load_dword_mask == 0) return VmAccessOutcome::Complete; + assert( + (d.atomic_op == AtomicOp::NONE || (~d.load_dword_mask & ((1u << d.num_dwords) - 1)) == 0) && + "a scalar atomic's mask covers its whole element or nothing"); if (wf.address_space()) { const bool may_refresh_unready = d.translated.initialized && d.translated.access && !d.translated.access->info().ready && diff --git a/emulation/rocjitsu/tests/scalar_atomic_test.cpp b/emulation/rocjitsu/tests/scalar_atomic_test.cpp index c318d944b5bd..e180d69e340e 100644 --- a/emulation/rocjitsu/tests/scalar_atomic_test.cpp +++ b/emulation/rocjitsu/tests/scalar_atomic_test.cpp @@ -89,9 +89,15 @@ class CachedScalarAtomicBacking final : public ScalarAtomicBacking { enum class MemoryMode { Direct, CachedVm, TranslatedVm }; +// S_BUFFER_ATOMIC_* opcodes sit 64 below their S_ATOMIC_* forms. +constexpr uint16_t buffer_atomic_opcode(uint16_t opcode) { return opcode - 64; } +static_assert(buffer_atomic_opcode(cdna3::kSAtomicSwapSmem) == cdna3::kSBufferAtomicSwapSmem); +static_assert(buffer_atomic_opcode(cdna3::kSAtomicDecX2Smem) == cdna3::kSBufferAtomicDecX2Smem); + class ScalarAtomicTest : public ::testing::TestWithParam> { protected: static constexpr uint64_t kAddress = 0x1000; + static constexpr uint32_t kRecords = 0x100; static constexpr uint64_t kProgram = 0x100000; static constexpr uint32_t kNop = cdna3::build_sopp(cdna3::kSNopSopp)[0]; @@ -128,8 +134,7 @@ class ScalarAtomicTest : public ::testing::TestWithParamset_process_id(7); wf->set_address_space(space); } - write_sgpr(0, kAddress); - write_sgpr(1, 0); + write_descriptor(kRecords); } void TearDown() override { @@ -139,6 +144,14 @@ class ScalarAtomicTest : public ::testing::TestWithParam(GetParam()); } + // s[0:1] is also the global atomic address. Buffer atomics ignore V# dword 3. + void write_descriptor(uint32_t records, uint32_t stride = 0) { + write_sgpr(0, kAddress); + write_sgpr(1, stride << 16); + write_sgpr(2, records); + write_sgpr(3, 0xffffffff); + } + void write_sgpr(unsigned reg, uint32_t value) { cu->write_sgpr(wf->sgpr_alloc().base + reg, value); } @@ -257,29 +270,31 @@ TEST_P(ScalarAtomicTest, ReadModifyWriteAndOptionalReturn) { {cdna3::kSAtomicDecX2Smem, 2, 0, 0x1'2345'6789, 0x1'2345'6789}, {cdna3::kSAtomicDecX2Smem, 2, ~uint64_t{0}, ~uint64_t{0}, ~uint64_t{1}}, }); - for (const auto &[opcode, dwords, old, data, expected] : cases) { - for (uint8_t glc : {0, 1}) { - SCOPED_TRACE(testing::Message() - << "opcode=" << opcode << " old=" << old << " data=" << data << " glc=" << +glc); - const uint64_t guard_address = kAddress + dwords * 4; - const unsigned guard_sgpr = 4 + dwords; - write_memory(kAddress, dwords, old); - memory.write32(guard_address, 0xdeadbeef); - write_sgprs(4, dwords, data); - write_sgpr(guard_sgpr, 0x12345678); - wf->write_scc(1); - auto inst = prepare({.sdata = 4, .glc = glc, .imm = 1}, opcode); - ASSERT_NE(inst, nullptr); - EXPECT_EQ(read_memory(kAddress, dwords), old); - EXPECT_EQ(read_sgprs(4, dwords), data); - ASSERT_EQ(pipeline->issue(inst.release(), *wf), VmAccessOutcome::Complete); - EXPECT_EQ(read_memory(kAddress, dwords), expected); - EXPECT_EQ(read_sgprs(4, dwords), glc ? old : data); - EXPECT_EQ(memory.read32(guard_address), 0xdeadbeef); - EXPECT_EQ(read_sgpr(guard_sgpr), 0x12345678); - EXPECT_EQ(wf->read_scc(), 1u); - EXPECT_EQ(wf->exec(), 0u); - EXPECT_TRUE(wf->wait_counters().empty()); + for (const auto &[global_opcode, dwords, old, data, expected] : cases) { + for (const uint16_t opcode : {global_opcode, buffer_atomic_opcode(global_opcode)}) { + for (uint8_t glc : {0, 1}) { + SCOPED_TRACE(testing::Message() << "opcode=" << opcode << " old=" << old << " data=" << data + << " glc=" << +glc); + const uint64_t guard_address = kAddress + dwords * 4; + const unsigned guard_sgpr = 4 + dwords; + write_memory(kAddress, dwords, old); + memory.write32(guard_address, 0xdeadbeef); + write_sgprs(4, dwords, data); + write_sgpr(guard_sgpr, 0x12345678); + wf->write_scc(1); + auto inst = prepare({.sdata = 4, .glc = glc, .imm = 1}, opcode); + ASSERT_NE(inst, nullptr); + EXPECT_EQ(read_memory(kAddress, dwords), old); + EXPECT_EQ(read_sgprs(4, dwords), data); + ASSERT_EQ(pipeline->issue(inst.release(), *wf), VmAccessOutcome::Complete); + EXPECT_EQ(read_memory(kAddress, dwords), expected); + EXPECT_EQ(read_sgprs(4, dwords), glc ? old : data); + EXPECT_EQ(memory.read32(guard_address), 0xdeadbeef); + EXPECT_EQ(read_sgpr(guard_sgpr), 0x12345678); + EXPECT_EQ(wf->read_scc(), 1u); + EXPECT_EQ(wf->exec(), 0u); + EXPECT_TRUE(wf->wait_counters().empty()); + } } } } @@ -302,37 +317,39 @@ TEST_P(ScalarAtomicTest, CompareSwapReturnsOldAndPreservesCompare) { {cdna3::kSAtomicCmpswapX2Smem, 2, 0x1'0000'0005, 0x2'0000'0009, 0x3'0000'0005, 0x1'0000'0005}, {cdna3::kSAtomicCmpswapX2Smem, 2, 0x1'0000'0005, 0x2'0000'0009, 0x1'0000'0006, 0x1'0000'0005}, }); - for (const auto &[opcode, dwords, old, source, compare, expected] : cases) { - for (uint8_t glc : {0, 1}) { - SCOPED_TRACE(testing::Message() << "opcode=" << opcode << " old=" << old - << " compare=" << compare << " glc=" << +glc); - const uint64_t guard_address = kAddress + dwords * 4; - const unsigned compare_sgpr = 4 + dwords; - const unsigned guard_sgpr = 4 + 2 * dwords; - write_memory(kAddress, dwords, old); - memory.write32(guard_address, 0xdeadbeef); - write_sgprs(4, dwords, source); - write_sgprs(compare_sgpr, dwords, compare); - write_sgpr(guard_sgpr, 0x12345678); - if (backing) { - backing->update_attempts = 0; - backing->mutations = 0; - } - auto inst = prepare({.sdata = 4, .glc = glc, .imm = 1}, opcode); - ASSERT_NE(inst, nullptr); - ASSERT_EQ(pipeline->issue(inst.release(), *wf), VmAccessOutcome::Complete); - EXPECT_EQ(read_memory(kAddress, dwords), expected); - EXPECT_EQ(read_sgprs(4, dwords), glc ? old : source); - EXPECT_EQ(read_sgprs(compare_sgpr, dwords), compare); - EXPECT_EQ(memory.read32(guard_address), 0xdeadbeef); - EXPECT_EQ(read_sgpr(guard_sgpr), 0x12345678); - EXPECT_TRUE(wf->wait_counters().empty()); - if (backing) { - // A translated mismatch skips the compare/exchange, as for vector - // atomics. L2 always writes the line back. - const unsigned writes = old == compare || memory_mode() == MemoryMode::CachedVm ? 1u : 0u; - EXPECT_EQ(backing->update_attempts, writes); - EXPECT_EQ(backing->mutations, writes); + for (const auto &[global_opcode, dwords, old, source, compare, expected] : cases) { + for (const uint16_t opcode : {global_opcode, buffer_atomic_opcode(global_opcode)}) { + for (uint8_t glc : {0, 1}) { + SCOPED_TRACE(testing::Message() << "opcode=" << opcode << " old=" << old + << " compare=" << compare << " glc=" << +glc); + const uint64_t guard_address = kAddress + dwords * 4; + const unsigned compare_sgpr = 4 + dwords; + const unsigned guard_sgpr = 4 + 2 * dwords; + write_memory(kAddress, dwords, old); + memory.write32(guard_address, 0xdeadbeef); + write_sgprs(4, dwords, source); + write_sgprs(compare_sgpr, dwords, compare); + write_sgpr(guard_sgpr, 0x12345678); + if (backing) { + backing->update_attempts = 0; + backing->mutations = 0; + } + auto inst = prepare({.sdata = 4, .glc = glc, .imm = 1}, opcode); + ASSERT_NE(inst, nullptr); + ASSERT_EQ(pipeline->issue(inst.release(), *wf), VmAccessOutcome::Complete); + EXPECT_EQ(read_memory(kAddress, dwords), expected); + EXPECT_EQ(read_sgprs(4, dwords), glc ? old : source); + EXPECT_EQ(read_sgprs(compare_sgpr, dwords), compare); + EXPECT_EQ(memory.read32(guard_address), 0xdeadbeef); + EXPECT_EQ(read_sgpr(guard_sgpr), 0x12345678); + EXPECT_TRUE(wf->wait_counters().empty()); + if (backing) { + // A translated mismatch skips the compare/exchange, as for vector + // atomics. L2 always writes the line back. + const unsigned writes = old == compare || memory_mode() == MemoryMode::CachedVm ? 1u : 0u; + EXPECT_EQ(backing->update_attempts, writes); + EXPECT_EQ(backing->mutations, writes); + } } } } @@ -479,26 +496,97 @@ TEST_P(ScalarAtomicTest, UnavailableAtomicRetainsCounterAndUpdatesExactlyOnce) { TEST_P(ScalarAtomicTest, MisalignedTwoDwordAtomicFaultsWithoutUpdate) { // On gfx90a, S_ATOMIC_*_X2 at an address that is 4 mod 8 raises a memory - // violation and leaves both neighboring qwords unchanged. - memory.write64(kAddress, 0x5'0000'0005); - memory.write64(kAddress + 8, 0x5'0000'0005); - write_sgprs(4, 2, 9); - auto inst = prepare({.sdata = 4, .glc = 1, .imm = 1, .offset = 4}, cdna3::kSAtomicDecX2Smem); - ASSERT_NE(inst, nullptr); - EXPECT_EQ(pipeline->issue(inst.release(), *wf), VmAccessOutcome::Malformed); - EXPECT_TRUE(wf->wait_counters().empty()); - EXPECT_EQ(memory.read64(kAddress), 0x5'0000'0005u); - EXPECT_EQ(memory.read64(kAddress + 8), 0x5'0000'0005u); - EXPECT_EQ(read_sgprs(4, 2), 9u); - if (backing) { - EXPECT_EQ(backing->load_attempts, 0u); - EXPECT_EQ(backing->update_attempts, 0u); - // GpuVmAccess reports the violation to the runtime. - EXPECT_EQ(vm_faults, (std::vector>{ - {kAddress + 4, VmAccessKind::Atomic}})); - } else { - // Physical memory has no fault reporter. - EXPECT_TRUE(vm_faults.empty()); + // violation and leaves both neighboring qwords unchanged. The buffer form + // faults even with NUM_RECORDS = 0: alignment is checked before bounds. + write_descriptor(/*records=*/0); + for (const uint16_t opcode : + {cdna3::kSAtomicDecX2Smem, buffer_atomic_opcode(cdna3::kSAtomicDecX2Smem)}) { + SCOPED_TRACE(opcode); + vm_faults.clear(); + memory.write64(kAddress, 0x5'0000'0005); + memory.write64(kAddress + 8, 0x5'0000'0005); + write_sgprs(4, 2, 9); + auto inst = prepare({.sdata = 4, .glc = 1, .imm = 1, .offset = 4}, opcode); + ASSERT_NE(inst, nullptr); + EXPECT_EQ(pipeline->issue(inst.release(), *wf), VmAccessOutcome::Malformed); + EXPECT_TRUE(wf->wait_counters().empty()); + EXPECT_EQ(memory.read64(kAddress), 0x5'0000'0005u); + EXPECT_EQ(memory.read64(kAddress + 8), 0x5'0000'0005u); + EXPECT_EQ(read_sgprs(4, 2), 9u); + if (backing) { + EXPECT_EQ(backing->load_attempts, 0u); + EXPECT_EQ(backing->update_attempts, 0u); + // GpuVmAccess reports the violation to the runtime. + EXPECT_EQ(vm_faults, (std::vector>{ + {kAddress + 4, VmAccessKind::Atomic}})); + } else { + // Physical memory has no fault reporter. + EXPECT_TRUE(vm_faults.empty()); + } + } +} + +struct BufferBoundsCase { + uint16_t opcode; + uint32_t offset; + uint32_t soffset; + uint32_t stride; + uint32_t records; + bool in_range; +}; + +// gfx90a results for S_BUFFER_ATOMIC_INC[_X2]. The element must lie wholly below +// STRIDE ? NUM_RECORDS * STRIDE : NUM_RECORDS bytes; otherwise memory is +// untouched and GLC returns zero. +TEST_P(ScalarAtomicTest, BufferAtomicRunsOnlyWhenWhollyInRange) { + constexpr auto cases = std::to_array({ + {cdna3::kSBufferAtomicIncSmem, 12, 0, 0, 12, false}, + {cdna3::kSBufferAtomicIncSmem, 12, 0, 0, 13, false}, + {cdna3::kSBufferAtomicIncSmem, 12, 0, 0, 16, true}, + {cdna3::kSBufferAtomicIncSmem, 0x100, 0, 0, 16, false}, + {cdna3::kSBufferAtomicIncSmem, 0, 0, 0, 0, false}, + {cdna3::kSBufferAtomicIncX2Smem, 16, 0, 0, 20, false}, + {cdna3::kSBufferAtomicIncX2Smem, 16, 0, 0, 23, false}, + {cdna3::kSBufferAtomicIncX2Smem, 16, 0, 0, 24, true}, + // Stride scales the bound but not the address. + {cdna3::kSBufferAtomicIncX2Smem, 16, 0, 8, 2, false}, + {cdna3::kSBufferAtomicIncX2Smem, 16, 0, 8, 3, true}, + // SOFFSET is part of the checked offset. + {cdna3::kSBufferAtomicIncSmem, 0, 12, 0, 16, true}, + {cdna3::kSBufferAtomicIncSmem, 0, 16, 0, 16, false}, + }); + for (const auto &[opcode, offset, soffset, stride, records, in_range] : cases) { + for (uint8_t glc : {0, 1}) { + SCOPED_TRACE(testing::Message() + << "opcode=" << opcode << " offset=" << offset << " soffset=" << soffset + << " stride=" << stride << " records=" << records << " glc=" << +glc); + const uint32_t dwords = opcode == cdna3::kSBufferAtomicIncX2Smem ? 2 : 1; + const uint64_t address = kAddress + offset + soffset; + write_descriptor(records, stride); + write_sgpr(8, soffset); + write_memory(address, dwords, 6); + write_sgprs(4, dwords, 7); + if (backing) { + backing->load_attempts = 0; + backing->update_attempts = 0; + } + auto inst = prepare({.sdata = 4, + .soffset_en = static_cast(soffset != 0), + .glc = glc, + .imm = 1, + .offset = offset, + .soffset = 8}, + opcode); + ASSERT_NE(inst, nullptr); + ASSERT_EQ(pipeline->issue(inst.release(), *wf), VmAccessOutcome::Complete); + EXPECT_EQ(read_memory(address, dwords), in_range ? 7u : 6u); + EXPECT_EQ(read_sgprs(4, dwords), !glc ? 7u : in_range ? 6u : 0u); + EXPECT_TRUE(wf->wait_counters().empty()); + if (backing && !in_range) { + EXPECT_EQ(backing->load_attempts, 0u); + EXPECT_EQ(backing->update_attempts, 0u); + } + } } } From 51e09d7799a7d27ee37e803a62128f8ee7914b26 Mon Sep 17 00:00:00 2001 From: Allyson Cauble-Chantrenne Date: Wed, 4 Nov 2026 08:01:54 -0600 Subject: [PATCH 2/5] [rocjitsu][amdisa] Address reviewer comments * `s_buffer_atomics` get `modifiers.buffer_resource = &sbase` and `scalar_buffer_words = 3` like buffer loads. * Prevent future additions to `s_buffer_atomic` from being parsed as something else * Test change * Comment clarifications --- emulation/rocjitsu/docs/vm-design.md | 22 ++-- .../lib/python/amdisa/codegen/_generator.py | 2 +- .../rocjitsu/lib/python/amdisa/semantics.py | 11 +- .../python/amdisa/tests/test_sema_derive.py | 7 +- .../isa/arch/amdgpu/generated/cdna1/smem.h | 104 ++++++++++++++++++ .../isa/arch/amdgpu/generated/cdna2/smem.h | 104 ++++++++++++++++++ .../isa/arch/amdgpu/generated/cdna3/smem.h | 104 ++++++++++++++++++ .../isa/arch/amdgpu/generated/cdna4/smem.h | 104 ++++++++++++++++++ .../isa/arch/amdgpu/shared/addr_calc_scalar.h | 16 ++- .../src/rocjitsu/vm/amdgpu/mem_state.h | 5 +- .../tests/memory_wait_scoreboard_test.cpp | 42 +++++++ .../rocjitsu/tests/scalar_atomic_test.cpp | 10 +- 12 files changed, 500 insertions(+), 31 deletions(-) diff --git a/emulation/rocjitsu/docs/vm-design.md b/emulation/rocjitsu/docs/vm-design.md index de16737b5197..0ee40db927bb 100644 --- a/emulation/rocjitsu/docs/vm-design.md +++ b/emulation/rocjitsu/docs/vm-design.md @@ -563,16 +563,18 @@ SC0/GLC is set. The L1 line is invalidated after the atomic to prevent stale reads. Supported integer atomics: swap, cmpswap, add, sub, smin/umin, smax/umax, and, or, xor, inc, dec. -**Scalar atomics** (`s_atomic_*` and `s_atomic_*_x2`, CDNA1–4) bypass the -L1 scalar cache and perform read-modify-write at L2; the atomic boundary's -coherence epoch makes stale K$ lines refetch. The old value is returned to -SDATA when GLC is set. Supported: swap, cmpswap, add, sub, smin/umin, -smax/umax, and, or, xor, inc, dec. Compare-swap reads the source and then -the compare value from SDATA and returns only the old value, to the low half. -`s_buffer_atomic_*` support the same operations. Their address is the V# base -plus the offset; stride only scales the bound. An atomic whose element is not -wholly in range performs no access and returns zero when GLC is set. A -misaligned `_x2` atomic faults even when out of range. +**Scalar atomics** (`s_atomic_*`, `s_buffer_atomic_*` and their `_x2` forms, +CDNA1–4) bypass the L1 scalar cache and perform read-modify-write at L2; the +atomic boundary's coherence epoch makes stale K$ lines refetch. The old value +is returned to SDATA when GLC is set. Supported: swap, cmpswap, add, sub, +smin/umin, smax/umax, and, or, xor, inc, dec. Compare-swap reads the source +and then the compare value from SDATA and returns only the old value, to the +low half. A buffer atomic's address is the V# base plus the offset; stride +only scales the bound. A buffer atomic whose element is not wholly in range +performs no access and returns zero when GLC is set. A misaligned `_x2` buffer +atomic faults even when out of range. The CDNA ISA manuals do not describe +these bounds and fault-order rules; they are observed on gfx90a and modeled +the same way on every CDNA target. **Cache management instructions:** diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py index 63aa1b2d86d3..f846db06999d 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py @@ -11540,7 +11540,7 @@ def gen_insts(self) -> None: f'modifiers.block_data = &{self.isa_spec.profile.flat_store_src_field};' ) if is_smem and inst.name.startswith( - ('S_BUFFER_LOAD_', 'S_BUFFER_STORE_') + ('S_BUFFER_LOAD_', 'S_BUFFER_STORE_', 'S_BUFFER_ATOMIC_') ): words = ( 2 diff --git a/emulation/rocjitsu/lib/python/amdisa/semantics.py b/emulation/rocjitsu/lib/python/amdisa/semantics.py index 982c8427d1e7..07ab527b467e 100644 --- a/emulation/rocjitsu/lib/python/amdisa/semantics.py +++ b/emulation/rocjitsu/lib/python/amdisa/semantics.py @@ -1878,9 +1878,8 @@ def _derive_vop3p(name: str) -> InstructionSemantics | None: } # S_ATOMIC_* and S_BUFFER_ATOMIC_* operations the CDNA ISAs define and the -# scalar pipeline supports. -# Other flat atomic operations do not derive smem_atomic, even when the flat -# tables know them. +# scalar pipeline supports, spelled as in the mnemonic without _X2. Any other +# scalar atomic derives nop, even when the flat tables know its operation. _SMEM_ATOMIC_OPS = frozenset( { 'swap', @@ -1935,9 +1934,8 @@ def _derive_smem(name: str) -> InstructionSemantics | None: is_x2 = suffix.endswith('_X2') if is_x2: suffix = suffix[:-3] - info = _derive_flat_atomic_info(suffix, is_x2) - if info and info[0] in _SMEM_ATOMIC_OPS: - op, elem_size, data_dw_actual = info + if suffix.lower() in _SMEM_ATOMIC_OPS: + op, elem_size, data_dw_actual = _derive_flat_atomic_info(suffix, is_x2) return InstructionSemantics( name, 'smem_atomic', @@ -1945,6 +1943,7 @@ def _derive_smem(name: str) -> InstructionSemantics | None: elem_size=elem_size, num_elems=data_dw_actual, ) + return InstructionSemantics(name, 'nop') is_store = '_STORE_' in upper or '_SCRATCH_STORE_' in upper if not is_store: diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py index 4f7a95a1bff9..fe2e9bda6f08 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py @@ -2399,8 +2399,9 @@ def test_implemented_scalar_atomics_derive_operation_and_widths(self, name): assert sem.elem_size == elem_size assert sem.num_elems == num_elems - # No CDNA ISA defines these. The flat atomic tables know their operations, - # so they pin that scalar support comes from the allowlist, not that table. + # No CDNA ISA defines these. They pin that scalar support comes from the + # allowlist: the flat atomic tables know most of these operations, SUB_F32 + # strips to an allowlisted SUB, and ADD_U16 ends in a narrow load suffix. @pytest.mark.parametrize( 'name', [ @@ -2411,6 +2412,8 @@ def test_implemented_scalar_atomics_derive_operation_and_widths(self, name): 'S_ATOMIC_COND_SUB_U32', 'S_BUFFER_ATOMIC_ADD_F32', 'S_BUFFER_ATOMIC_FMIN', + 'S_BUFFER_ATOMIC_SUB_F32', + 'S_BUFFER_ATOMIC_ADD_U16', ], ) def test_scalar_atomic_operations_outside_allowlist_stay_nop(self, name): diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/smem.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/smem.h index 30fffc147d13..27b32e33e442 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/smem.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/smem.h @@ -342,6 +342,10 @@ class SBufferAtomicSwapSmem : public Smem { public: SBufferAtomicSwapSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -351,6 +355,10 @@ class SBufferAtomicCmpswapSmem : public Smem { public: SBufferAtomicCmpswapSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sdata_return; Operand sbase; @@ -361,6 +369,10 @@ class SBufferAtomicAddSmem : public Smem { public: SBufferAtomicAddSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -370,6 +382,10 @@ class SBufferAtomicSubSmem : public Smem { public: SBufferAtomicSubSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -379,6 +395,10 @@ class SBufferAtomicSminSmem : public Smem { public: SBufferAtomicSminSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -388,6 +408,10 @@ class SBufferAtomicUminSmem : public Smem { public: SBufferAtomicUminSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -397,6 +421,10 @@ class SBufferAtomicSmaxSmem : public Smem { public: SBufferAtomicSmaxSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -406,6 +434,10 @@ class SBufferAtomicUmaxSmem : public Smem { public: SBufferAtomicUmaxSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -415,6 +447,10 @@ class SBufferAtomicAndSmem : public Smem { public: SBufferAtomicAndSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -424,6 +460,10 @@ class SBufferAtomicOrSmem : public Smem { public: SBufferAtomicOrSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -433,6 +473,10 @@ class SBufferAtomicXorSmem : public Smem { public: SBufferAtomicXorSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -442,6 +486,10 @@ class SBufferAtomicIncSmem : public Smem { public: SBufferAtomicIncSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -451,6 +499,10 @@ class SBufferAtomicDecSmem : public Smem { public: SBufferAtomicDecSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -460,6 +512,10 @@ class SBufferAtomicSwapX2Smem : public Smem { public: SBufferAtomicSwapX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -469,6 +525,10 @@ class SBufferAtomicCmpswapX2Smem : public Smem { public: SBufferAtomicCmpswapX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sdata_return; Operand sbase; @@ -479,6 +539,10 @@ class SBufferAtomicAddX2Smem : public Smem { public: SBufferAtomicAddX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -488,6 +552,10 @@ class SBufferAtomicSubX2Smem : public Smem { public: SBufferAtomicSubX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -497,6 +565,10 @@ class SBufferAtomicSminX2Smem : public Smem { public: SBufferAtomicSminX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -506,6 +578,10 @@ class SBufferAtomicUminX2Smem : public Smem { public: SBufferAtomicUminX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -515,6 +591,10 @@ class SBufferAtomicSmaxX2Smem : public Smem { public: SBufferAtomicSmaxX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -524,6 +604,10 @@ class SBufferAtomicUmaxX2Smem : public Smem { public: SBufferAtomicUmaxX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -533,6 +617,10 @@ class SBufferAtomicAndX2Smem : public Smem { public: SBufferAtomicAndX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -542,6 +630,10 @@ class SBufferAtomicOrX2Smem : public Smem { public: SBufferAtomicOrX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -551,6 +643,10 @@ class SBufferAtomicXorX2Smem : public Smem { public: SBufferAtomicXorX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -560,6 +656,10 @@ class SBufferAtomicIncX2Smem : public Smem { public: SBufferAtomicIncX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -569,6 +669,10 @@ class SBufferAtomicDecX2Smem : public Smem { public: SBufferAtomicDecX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/smem.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/smem.h index 4ed82e7b7f19..4fbd45121a7c 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/smem.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/smem.h @@ -342,6 +342,10 @@ class SBufferAtomicSwapSmem : public Smem { public: SBufferAtomicSwapSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -351,6 +355,10 @@ class SBufferAtomicCmpswapSmem : public Smem { public: SBufferAtomicCmpswapSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sdata_return; Operand sbase; @@ -361,6 +369,10 @@ class SBufferAtomicAddSmem : public Smem { public: SBufferAtomicAddSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -370,6 +382,10 @@ class SBufferAtomicSubSmem : public Smem { public: SBufferAtomicSubSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -379,6 +395,10 @@ class SBufferAtomicSminSmem : public Smem { public: SBufferAtomicSminSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -388,6 +408,10 @@ class SBufferAtomicUminSmem : public Smem { public: SBufferAtomicUminSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -397,6 +421,10 @@ class SBufferAtomicSmaxSmem : public Smem { public: SBufferAtomicSmaxSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -406,6 +434,10 @@ class SBufferAtomicUmaxSmem : public Smem { public: SBufferAtomicUmaxSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -415,6 +447,10 @@ class SBufferAtomicAndSmem : public Smem { public: SBufferAtomicAndSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -424,6 +460,10 @@ class SBufferAtomicOrSmem : public Smem { public: SBufferAtomicOrSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -433,6 +473,10 @@ class SBufferAtomicXorSmem : public Smem { public: SBufferAtomicXorSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -442,6 +486,10 @@ class SBufferAtomicIncSmem : public Smem { public: SBufferAtomicIncSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -451,6 +499,10 @@ class SBufferAtomicDecSmem : public Smem { public: SBufferAtomicDecSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -460,6 +512,10 @@ class SBufferAtomicSwapX2Smem : public Smem { public: SBufferAtomicSwapX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -469,6 +525,10 @@ class SBufferAtomicCmpswapX2Smem : public Smem { public: SBufferAtomicCmpswapX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sdata_return; Operand sbase; @@ -479,6 +539,10 @@ class SBufferAtomicAddX2Smem : public Smem { public: SBufferAtomicAddX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -488,6 +552,10 @@ class SBufferAtomicSubX2Smem : public Smem { public: SBufferAtomicSubX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -497,6 +565,10 @@ class SBufferAtomicSminX2Smem : public Smem { public: SBufferAtomicSminX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -506,6 +578,10 @@ class SBufferAtomicUminX2Smem : public Smem { public: SBufferAtomicUminX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -515,6 +591,10 @@ class SBufferAtomicSmaxX2Smem : public Smem { public: SBufferAtomicSmaxX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -524,6 +604,10 @@ class SBufferAtomicUmaxX2Smem : public Smem { public: SBufferAtomicUmaxX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -533,6 +617,10 @@ class SBufferAtomicAndX2Smem : public Smem { public: SBufferAtomicAndX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -542,6 +630,10 @@ class SBufferAtomicOrX2Smem : public Smem { public: SBufferAtomicOrX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -551,6 +643,10 @@ class SBufferAtomicXorX2Smem : public Smem { public: SBufferAtomicXorX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -560,6 +656,10 @@ class SBufferAtomicIncX2Smem : public Smem { public: SBufferAtomicIncX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -569,6 +669,10 @@ class SBufferAtomicDecX2Smem : public Smem { public: SBufferAtomicDecX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/smem.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/smem.h index c2e8c72fabce..bdf18e9b886b 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/smem.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/smem.h @@ -342,6 +342,10 @@ class SBufferAtomicSwapSmem : public Smem { public: SBufferAtomicSwapSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -351,6 +355,10 @@ class SBufferAtomicCmpswapSmem : public Smem { public: SBufferAtomicCmpswapSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sdata_return; Operand sbase; @@ -361,6 +369,10 @@ class SBufferAtomicAddSmem : public Smem { public: SBufferAtomicAddSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -370,6 +382,10 @@ class SBufferAtomicSubSmem : public Smem { public: SBufferAtomicSubSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -379,6 +395,10 @@ class SBufferAtomicSminSmem : public Smem { public: SBufferAtomicSminSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -388,6 +408,10 @@ class SBufferAtomicUminSmem : public Smem { public: SBufferAtomicUminSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -397,6 +421,10 @@ class SBufferAtomicSmaxSmem : public Smem { public: SBufferAtomicSmaxSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -406,6 +434,10 @@ class SBufferAtomicUmaxSmem : public Smem { public: SBufferAtomicUmaxSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -415,6 +447,10 @@ class SBufferAtomicAndSmem : public Smem { public: SBufferAtomicAndSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -424,6 +460,10 @@ class SBufferAtomicOrSmem : public Smem { public: SBufferAtomicOrSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -433,6 +473,10 @@ class SBufferAtomicXorSmem : public Smem { public: SBufferAtomicXorSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -442,6 +486,10 @@ class SBufferAtomicIncSmem : public Smem { public: SBufferAtomicIncSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -451,6 +499,10 @@ class SBufferAtomicDecSmem : public Smem { public: SBufferAtomicDecSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -460,6 +512,10 @@ class SBufferAtomicSwapX2Smem : public Smem { public: SBufferAtomicSwapX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -469,6 +525,10 @@ class SBufferAtomicCmpswapX2Smem : public Smem { public: SBufferAtomicCmpswapX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sdata_return; Operand sbase; @@ -479,6 +539,10 @@ class SBufferAtomicAddX2Smem : public Smem { public: SBufferAtomicAddX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -488,6 +552,10 @@ class SBufferAtomicSubX2Smem : public Smem { public: SBufferAtomicSubX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -497,6 +565,10 @@ class SBufferAtomicSminX2Smem : public Smem { public: SBufferAtomicSminX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -506,6 +578,10 @@ class SBufferAtomicUminX2Smem : public Smem { public: SBufferAtomicUminX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -515,6 +591,10 @@ class SBufferAtomicSmaxX2Smem : public Smem { public: SBufferAtomicSmaxX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -524,6 +604,10 @@ class SBufferAtomicUmaxX2Smem : public Smem { public: SBufferAtomicUmaxX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -533,6 +617,10 @@ class SBufferAtomicAndX2Smem : public Smem { public: SBufferAtomicAndX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -542,6 +630,10 @@ class SBufferAtomicOrX2Smem : public Smem { public: SBufferAtomicOrX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -551,6 +643,10 @@ class SBufferAtomicXorX2Smem : public Smem { public: SBufferAtomicXorX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -560,6 +656,10 @@ class SBufferAtomicIncX2Smem : public Smem { public: SBufferAtomicIncX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -569,6 +669,10 @@ class SBufferAtomicDecX2Smem : public Smem { public: SBufferAtomicDecX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/smem.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/smem.h index e5fa032f1d24..9c8bb24a5f68 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/smem.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/smem.h @@ -342,6 +342,10 @@ class SBufferAtomicSwapSmem : public Smem { public: SBufferAtomicSwapSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -351,6 +355,10 @@ class SBufferAtomicCmpswapSmem : public Smem { public: SBufferAtomicCmpswapSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sdata_return; Operand sbase; @@ -361,6 +369,10 @@ class SBufferAtomicAddSmem : public Smem { public: SBufferAtomicAddSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -370,6 +382,10 @@ class SBufferAtomicSubSmem : public Smem { public: SBufferAtomicSubSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -379,6 +395,10 @@ class SBufferAtomicSminSmem : public Smem { public: SBufferAtomicSminSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -388,6 +408,10 @@ class SBufferAtomicUminSmem : public Smem { public: SBufferAtomicUminSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -397,6 +421,10 @@ class SBufferAtomicSmaxSmem : public Smem { public: SBufferAtomicSmaxSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -406,6 +434,10 @@ class SBufferAtomicUmaxSmem : public Smem { public: SBufferAtomicUmaxSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -415,6 +447,10 @@ class SBufferAtomicAndSmem : public Smem { public: SBufferAtomicAndSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -424,6 +460,10 @@ class SBufferAtomicOrSmem : public Smem { public: SBufferAtomicOrSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -433,6 +473,10 @@ class SBufferAtomicXorSmem : public Smem { public: SBufferAtomicXorSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -442,6 +486,10 @@ class SBufferAtomicIncSmem : public Smem { public: SBufferAtomicIncSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -451,6 +499,10 @@ class SBufferAtomicDecSmem : public Smem { public: SBufferAtomicDecSmem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -460,6 +512,10 @@ class SBufferAtomicSwapX2Smem : public Smem { public: SBufferAtomicSwapX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -469,6 +525,10 @@ class SBufferAtomicCmpswapX2Smem : public Smem { public: SBufferAtomicCmpswapX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sdata_return; Operand sbase; @@ -479,6 +539,10 @@ class SBufferAtomicAddX2Smem : public Smem { public: SBufferAtomicAddX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -488,6 +552,10 @@ class SBufferAtomicSubX2Smem : public Smem { public: SBufferAtomicSubX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -497,6 +565,10 @@ class SBufferAtomicSminX2Smem : public Smem { public: SBufferAtomicSminX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -506,6 +578,10 @@ class SBufferAtomicUminX2Smem : public Smem { public: SBufferAtomicUminX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -515,6 +591,10 @@ class SBufferAtomicSmaxX2Smem : public Smem { public: SBufferAtomicSmaxX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -524,6 +604,10 @@ class SBufferAtomicUmaxX2Smem : public Smem { public: SBufferAtomicUmaxX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -533,6 +617,10 @@ class SBufferAtomicAndX2Smem : public Smem { public: SBufferAtomicAndX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -542,6 +630,10 @@ class SBufferAtomicOrX2Smem : public Smem { public: SBufferAtomicOrX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -551,6 +643,10 @@ class SBufferAtomicXorX2Smem : public Smem { public: SBufferAtomicXorX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -560,6 +656,10 @@ class SBufferAtomicIncX2Smem : public Smem { public: SBufferAtomicIncX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; @@ -569,6 +669,10 @@ class SBufferAtomicDecX2Smem : public Smem { public: SBufferAtomicDecX2Smem(const MachineInst *inst); void execute_impl(amdgpu::Wavefront &wf); + void amdgpu_register_modifiers(amdgpu::RegisterModifiers &modifiers) const override { + modifiers.buffer_resource = &sbase; + modifiers.scalar_buffer_words = 3; + } Operand sdata; Operand sbase; Operand soffset; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/addr_calc_scalar.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/addr_calc_scalar.h index 909e3593d995..1f2a87e76cc0 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/addr_calc_scalar.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/addr_calc_scalar.h @@ -28,8 +28,9 @@ namespace rocjitsu { namespace amdgpu { namespace addr_calc { -/// Mark each in-range DWORD separately so partial buffer loads zero only the -/// out-of-range destinations. Byte and halfword loads use one destination DWORD. +/// Set the mask bit of each DWORD that starts below @p bound and clear the rest. +/// A load returns zero for a clear DWORD and a store skips it. Byte and halfword +/// loads use one destination DWORD. inline void scalar_buffer_load_mask(ScalarMemState *state, int64_t offset, uint64_t bound) { if (!state) return; @@ -50,6 +51,11 @@ inline void scalar_buffer_load_mask(ScalarMemState *state, int64_t offset, uint6 /// is. GFX9 passes 3, so a DWORD must lie wholly in range, and RDNA4 passes its /// access alignment. RDNA1 through RDNA3.5 pass zero; their hardware rule is /// unverified. +/// +/// An atomic's mask is all set or all clear. An aligned atomic is accessed only +/// if every DWORD of its element is in range. A misaligned atomic keeps its full +/// mask so the access path faults, because alignment is checked before bounds. +/// The CDNA ISA manuals do not describe these rules; they are observed on gfx90a. inline std::optional scalar_buffer_address(Wavefront &wf, uint32_t selector, uint64_t base, int64_t offset, ScalarMemState *state, uint64_t align_mask, @@ -61,12 +67,10 @@ inline std::optional scalar_buffer_address(Wavefront &wf, uint32_t sel scalar_buffer_load_mask(state, offset, (*records * (stride ? stride : 1)) & ~size_align_mask); const uint64_t address = buffer_virtual_address(base + offset) & ~align_mask; if (state && state->atomic_op != AtomicOp::NONE) { - // An atomic accesses its whole element or nothing. Hardware checks atomic - // alignment before bounds, so a misaligned atomic keeps its full mask and - // faults in the access path. const uint16_t element_mask = (1u << state->num_dwords) - 1; const bool aligned = address % state->elem_size == 0; - state->load_dword_mask = state->load_dword_mask == element_mask || !aligned ? element_mask : 0; + state->load_dword_mask = + (state->load_dword_mask == element_mask || !aligned) ? element_mask : 0; } return address; } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/mem_state.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/mem_state.h index b4b04c927b00..3133609e0bf9 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/mem_state.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/mem_state.h @@ -95,8 +95,9 @@ class ScalarMemState : public DynamicInstState { AtomicOp atomic_op = AtomicOp::NONE; Mtype mtype = Mtype::RW; WaitCounterType wait_counter_type = WaitCounterType::LGKMCNT; - /// In-range dwords of a buffer access: masked load dwords read zero and masked - /// store dwords are skipped. An atomic's mask covers its whole element or is zero. + /// One bit per DWORD of a buffer access. A set bit selects the DWORD for + /// access; a clear bit makes a load return zero for it or skips it for a + /// store. An atomic's mask is all set or all clear. uint16_t load_dword_mask = 0xffff; uint32_t response_data[16] = {}; uint32_t store_data[16] = {}; diff --git a/emulation/rocjitsu/tests/memory_wait_scoreboard_test.cpp b/emulation/rocjitsu/tests/memory_wait_scoreboard_test.cpp index 9711b75c4ec7..fd40ffd9cdcf 100644 --- a/emulation/rocjitsu/tests/memory_wait_scoreboard_test.cpp +++ b/emulation/rocjitsu/tests/memory_wait_scoreboard_test.cpp @@ -5361,6 +5361,48 @@ TEST(MemoryWaitExecutionTest, ScalarBufferStoreChecksItsConsumedBaseWords) { } } +TEST(MemoryWaitExecutionTest, ScalarBufferAtomicChecksItsConsumedDescriptorWords) { + GpuMemory memory("smem_atomic_wait_memory"); + L2Cache l2("smem_atomic_wait_l2"); + ComputeUnitCore::Config config{}; + config.memory_wait_diagnostics = MemoryWaitDiagnostics::Warn; + config.arch = ROCJITSU_CODE_ARCH_CDNA4; + config.num_wf_slots = 1; + config.sgprs_per_wf = 106; + config.vgprs_per_wf = 32; + auto cu = ComputeUnitCore::create("smem_atomic_wait_cu", config, &memory, &l2); + auto *wf = cu->dispatch_wf(0, 0x100, 106, 32, 64); + ASSERT_NE(wf, nullptr); + auto decoder = Decoder::create(config.arch); + // s_buffer_atomic_inc s4, s[8:11], 0 glc + const auto words = + cdna4::build_smem(cdna4::kSBufferAtomicIncSmem, {.sbase = 4, .sdata = 4, .glc = 1, .imm = 1}); + util::StringDiagnostic error; + auto consumer = decoder->decode_window(words, 0, error.emitter()); + ASSERT_TRUE(consumer.succeeded()) << error.message(); + auto &state = wf->ensure_memory_wait_scoreboard(); + // The base and NUM_RECORDS are consumed; descriptor word 3 is unused. + for (uint8_t pending : {8, 9, 10, 11}) + for (bool waited : {false, true}) { + SCOPED_TRACE(unsigned(pending)); + SCOPED_TRACE(waited); + const auto producer_words = + cdna4::build_smem(cdna4::kSLoadDwordSmem, {.sdata = pending, .imm = 1}); + auto producer = decoder->decode_window(producer_words, 0, error.emitter()); + ASSERT_TRUE(producer.succeeded()) << error.message(); + state.clear(); + cu->track_memory_wait(*producer.value(), *wf); + ASSERT_EQ(state.events().size(), 1u); + if (waited) + state.wait(state.events().front().counter, 0); + unsigned reports = 0; + state.bind(0x200, &reports, + [](void *context, const auto &) { ++*static_cast(context); }); + state.check_instruction(*consumer.value(), *wf); + EXPECT_EQ(reports, pending < 11 && !waited ? 1u : 0u); + } +} + TEST(MemoryWaitFootprintTest, ScalarBufferDescriptorsRetainOnlyBackedAndConsumedWords) { for (auto arch : {ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_CDNA5}) for (unsigned sgprs : {62u, 63u, 64u, 65u, 66u}) { diff --git a/emulation/rocjitsu/tests/scalar_atomic_test.cpp b/emulation/rocjitsu/tests/scalar_atomic_test.cpp index e180d69e340e..5f45e18a577e 100644 --- a/emulation/rocjitsu/tests/scalar_atomic_test.cpp +++ b/emulation/rocjitsu/tests/scalar_atomic_test.cpp @@ -497,7 +497,8 @@ TEST_P(ScalarAtomicTest, UnavailableAtomicRetainsCounterAndUpdatesExactlyOnce) { TEST_P(ScalarAtomicTest, MisalignedTwoDwordAtomicFaultsWithoutUpdate) { // On gfx90a, S_ATOMIC_*_X2 at an address that is 4 mod 8 raises a memory // violation and leaves both neighboring qwords unchanged. The buffer form - // faults even with NUM_RECORDS = 0: alignment is checked before bounds. + // faults even with NUM_RECORDS = 0: alignment is checked before bounds. The + // CDNA ISA manuals do not describe this ordering. write_descriptor(/*records=*/0); for (const uint16_t opcode : {cdna3::kSAtomicDecX2Smem, buffer_atomic_opcode(cdna3::kSAtomicDecX2Smem)}) { @@ -535,7 +536,8 @@ struct BufferBoundsCase { bool in_range; }; -// gfx90a results for S_BUFFER_ATOMIC_INC[_X2]. The element must lie wholly below +// S_BUFFER_ATOMIC_INC[_X2] results observed on gfx90a; the CDNA ISA manuals do +// not describe them. The element must lie wholly below // STRIDE ? NUM_RECORDS * STRIDE : NUM_RECORDS bytes; otherwise memory is // untouched and GLC returns zero. TEST_P(ScalarAtomicTest, BufferAtomicRunsOnlyWhenWhollyInRange) { @@ -563,7 +565,7 @@ TEST_P(ScalarAtomicTest, BufferAtomicRunsOnlyWhenWhollyInRange) { const uint32_t dwords = opcode == cdna3::kSBufferAtomicIncX2Smem ? 2 : 1; const uint64_t address = kAddress + offset + soffset; write_descriptor(records, stride); - write_sgpr(8, soffset); + wf->set_m0(soffset); write_memory(address, dwords, 6); write_sgprs(4, dwords, 7); if (backing) { @@ -575,7 +577,7 @@ TEST_P(ScalarAtomicTest, BufferAtomicRunsOnlyWhenWhollyInRange) { .glc = glc, .imm = 1, .offset = offset, - .soffset = 8}, + .soffset = kLegacyM0Selector}, opcode); ASSERT_NE(inst, nullptr); ASSERT_EQ(pipeline->issue(inst.release(), *wf), VmAccessOutcome::Complete); From 06b1397072f3cb72c7b05a54fdb9065ff9b79191 Mon Sep 17 00:00:00 2001 From: Allyson Cauble-Chantrenne Date: Wed, 4 Nov 2026 12:47:37 -0600 Subject: [PATCH 3/5] [rocjitsu][amdisa] Code cleanup --- .../rocjitsu/lib/python/amdisa/semantics.py | 28 +++++++++---------- .../rocjitsu/tests/scalar_atomic_test.cpp | 5 +++- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/emulation/rocjitsu/lib/python/amdisa/semantics.py b/emulation/rocjitsu/lib/python/amdisa/semantics.py index 07ab527b467e..ecc2e6c89755 100644 --- a/emulation/rocjitsu/lib/python/amdisa/semantics.py +++ b/emulation/rocjitsu/lib/python/amdisa/semantics.py @@ -1882,19 +1882,19 @@ def _derive_vop3p(name: str) -> InstructionSemantics | None: # scalar atomic derives nop, even when the flat tables know its operation. _SMEM_ATOMIC_OPS = frozenset( { - 'swap', - 'cmpswap', - 'add', - 'sub', - 'smin', - 'umin', - 'smax', - 'umax', - 'and', - 'or', - 'xor', - 'inc', - 'dec', + 'SWAP', + 'CMPSWAP', + 'ADD', + 'SUB', + 'SMIN', + 'UMIN', + 'SMAX', + 'UMAX', + 'AND', + 'OR', + 'XOR', + 'INC', + 'DEC', } ) @@ -1934,7 +1934,7 @@ def _derive_smem(name: str) -> InstructionSemantics | None: is_x2 = suffix.endswith('_X2') if is_x2: suffix = suffix[:-3] - if suffix.lower() in _SMEM_ATOMIC_OPS: + if suffix in _SMEM_ATOMIC_OPS: op, elem_size, data_dw_actual = _derive_flat_atomic_info(suffix, is_x2) return InstructionSemantics( name, diff --git a/emulation/rocjitsu/tests/scalar_atomic_test.cpp b/emulation/rocjitsu/tests/scalar_atomic_test.cpp index 5f45e18a577e..d0477b6a215b 100644 --- a/emulation/rocjitsu/tests/scalar_atomic_test.cpp +++ b/emulation/rocjitsu/tests/scalar_atomic_test.cpp @@ -325,6 +325,9 @@ TEST_P(ScalarAtomicTest, CompareSwapReturnsOldAndPreservesCompare) { const uint64_t guard_address = kAddress + dwords * 4; const unsigned compare_sgpr = 4 + dwords; const unsigned guard_sgpr = 4 + 2 * dwords; + // The buffer bound fits only the memory element: CMPSWAP_X2 reads 16 + // bytes of SGPRs but accesses 8 bytes of memory. + write_descriptor(dwords * 4); write_memory(kAddress, dwords, old); memory.write32(guard_address, 0xdeadbeef); write_sgprs(4, dwords, source); @@ -582,7 +585,7 @@ TEST_P(ScalarAtomicTest, BufferAtomicRunsOnlyWhenWhollyInRange) { ASSERT_NE(inst, nullptr); ASSERT_EQ(pipeline->issue(inst.release(), *wf), VmAccessOutcome::Complete); EXPECT_EQ(read_memory(address, dwords), in_range ? 7u : 6u); - EXPECT_EQ(read_sgprs(4, dwords), !glc ? 7u : in_range ? 6u : 0u); + EXPECT_EQ(read_sgprs(4, dwords), !glc ? 7u : (in_range ? 6u : 0u)); EXPECT_TRUE(wf->wait_counters().empty()); if (backing && !in_range) { EXPECT_EQ(backing->load_attempts, 0u); From 6dd842d8567a0ee5f9c33e4b35001c673f0d256d Mon Sep 17 00:00:00 2001 From: Allyson Cauble-Chantrenne Date: Wed, 4 Nov 2026 13:20:33 -0600 Subject: [PATCH 4/5] [rocjitsu][amdisa] test cleanup --- .../rocjitsu/tests/scalar_atomic_test.cpp | 57 ++++++++++--------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/emulation/rocjitsu/tests/scalar_atomic_test.cpp b/emulation/rocjitsu/tests/scalar_atomic_test.cpp index d0477b6a215b..db212e988c6f 100644 --- a/emulation/rocjitsu/tests/scalar_atomic_test.cpp +++ b/emulation/rocjitsu/tests/scalar_atomic_test.cpp @@ -325,9 +325,10 @@ TEST_P(ScalarAtomicTest, CompareSwapReturnsOldAndPreservesCompare) { const uint64_t guard_address = kAddress + dwords * 4; const unsigned compare_sgpr = 4 + dwords; const unsigned guard_sgpr = 4 + 2 * dwords; - // The buffer bound fits only the memory element: CMPSWAP_X2 reads 16 - // bytes of SGPRs but accesses 8 bytes of memory. - write_descriptor(dwords * 4); + // The V# bound fits only the memory element: CMPSWAP_X2 reads 16 bytes + // of SGPRs but accesses 8 bytes of memory. + if (opcode != global_opcode) + write_descriptor(dwords * 4); write_memory(kAddress, dwords, old); memory.write32(guard_address, 0xdeadbeef); write_sgprs(4, dwords, source); @@ -713,29 +714,33 @@ TEST_P(ScalarAtomicTest, FailedCompareExchangeDoesNotWriteBackOrLeakCounter) { } TEST_P(ScalarAtomicTest, DecodedDependenciesFollowReturnBit) { - for (const auto &[opcode, dwords] : kDecrementForms) { - for (uint8_t glc : {0, 1}) { - SCOPED_TRACE(testing::Message() << "opcode=" << opcode << " glc=" << +glc); - const auto words = cdna3::build_smem(opcode, {.sdata = 4, .glc = glc, .imm = 1}); - std::unique_ptr inst(decode_valid(*decoder, words.data())); - ASSERT_NE(inst, nullptr); - const InstDefUse def_use(*inst, 64); - const RegisterRef sdata{RegClass::SGPR, 4, static_cast(dwords)}; - EXPECT_TRUE(def_use.uses.contains(sdata)); - EXPECT_TRUE(def_use.uses.contains({RegClass::SGPR, 0, 2})); - EXPECT_EQ(def_use.defs.contains(sdata), glc != 0); - EXPECT_EQ(inst->num_dst_operands(), glc ? 1 : 0); - - EXPECT_TRUE(inst->is_memory_wait_producer()); - const auto *issue = inst->amdgpu_memory_issue_info(); - ASSERT_NE(issue, nullptr); - EXPECT_FALSE(issue->exec_masked); - const auto obligations = issue->counter_obligations(); - ASSERT_EQ(obligations.size(), 1u); - EXPECT_EQ(obligations[0].wait_counter_type(), WaitCounterType::LGKMCNT); - EXPECT_EQ(obligations[0].completion_class(), MemoryCompletionClass::UNORDERED); - // The access width sets the token count, independent of GLC. - EXPECT_EQ(obligations[0].counter_increment(), dwords); + for (const auto &[global_opcode, dwords] : kDecrementForms) { + for (const uint16_t opcode : {global_opcode, buffer_atomic_opcode(global_opcode)}) { + // A buffer atomic's SBASE is a 128-bit V#, not a 64-bit address. + const uint8_t base_width = opcode == global_opcode ? 2 : 4; + for (uint8_t glc : {0, 1}) { + SCOPED_TRACE(testing::Message() << "opcode=" << opcode << " glc=" << +glc); + const auto words = cdna3::build_smem(opcode, {.sdata = 4, .glc = glc, .imm = 1}); + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + const InstDefUse def_use(*inst, 64); + const RegisterRef sdata{RegClass::SGPR, 4, static_cast(dwords)}; + EXPECT_TRUE(def_use.uses.contains(sdata)); + EXPECT_TRUE(def_use.uses.contains({RegClass::SGPR, 0, base_width})); + EXPECT_EQ(def_use.defs.contains(sdata), glc != 0); + EXPECT_EQ(inst->num_dst_operands(), glc ? 1 : 0); + + EXPECT_TRUE(inst->is_memory_wait_producer()); + const auto *issue = inst->amdgpu_memory_issue_info(); + ASSERT_NE(issue, nullptr); + EXPECT_FALSE(issue->exec_masked); + const auto obligations = issue->counter_obligations(); + ASSERT_EQ(obligations.size(), 1u); + EXPECT_EQ(obligations[0].wait_counter_type(), WaitCounterType::LGKMCNT); + EXPECT_EQ(obligations[0].completion_class(), MemoryCompletionClass::UNORDERED); + // The access width sets the token count, independent of GLC. + EXPECT_EQ(obligations[0].counter_increment(), dwords); + } } } } From 89949085170486b405875e4d27d5d94cee1fe347 Mon Sep 17 00:00:00 2001 From: Allyson Cauble-Chantrenne Date: Wed, 4 Nov 2026 15:19:37 -0600 Subject: [PATCH 5/5] [rocjitsu] test and doc tweaks --- emulation/rocjitsu/docs/vm-design.md | 7 +++++-- emulation/rocjitsu/tests/scalar_atomic_test.cpp | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/emulation/rocjitsu/docs/vm-design.md b/emulation/rocjitsu/docs/vm-design.md index 0ee40db927bb..55e6fc67a088 100644 --- a/emulation/rocjitsu/docs/vm-design.md +++ b/emulation/rocjitsu/docs/vm-design.md @@ -573,8 +573,11 @@ low half. A buffer atomic's address is the V# base plus the offset; stride only scales the bound. A buffer atomic whose element is not wholly in range performs no access and returns zero when GLC is set. A misaligned `_x2` buffer atomic faults even when out of range. The CDNA ISA manuals do not describe -these bounds and fault-order rules; they are observed on gfx90a and modeled -the same way on every CDNA target. +these bounds and fault-order rules. They are observed on gfx90a with +`s_buffer_atomic_inc[_x2]`, and the fault order also with +`s_buffer_atomic_dec_x2`. The other operations and CDNA targets are modeled +the same way by extrapolation, including `s_buffer_atomic_cmpswap_x2`, whose +bound covers its 8-byte memory element although SDATA holds 16 bytes. **Cache management instructions:** diff --git a/emulation/rocjitsu/tests/scalar_atomic_test.cpp b/emulation/rocjitsu/tests/scalar_atomic_test.cpp index db212e988c6f..e88c2d898f51 100644 --- a/emulation/rocjitsu/tests/scalar_atomic_test.cpp +++ b/emulation/rocjitsu/tests/scalar_atomic_test.cpp @@ -568,10 +568,12 @@ TEST_P(ScalarAtomicTest, BufferAtomicRunsOnlyWhenWhollyInRange) { << " stride=" << stride << " records=" << records << " glc=" << +glc); const uint32_t dwords = opcode == cdna3::kSBufferAtomicIncX2Smem ? 2 : 1; const uint64_t address = kAddress + offset + soffset; + // A nonzero upper SDATA dword shows a dropped _x2 that zeroes only one half. + const uint64_t data = dwords == 2 ? 0x1'0000'0007 : 7; write_descriptor(records, stride); wf->set_m0(soffset); write_memory(address, dwords, 6); - write_sgprs(4, dwords, 7); + write_sgprs(4, dwords, data); if (backing) { backing->load_attempts = 0; backing->update_attempts = 0; @@ -586,7 +588,7 @@ TEST_P(ScalarAtomicTest, BufferAtomicRunsOnlyWhenWhollyInRange) { ASSERT_NE(inst, nullptr); ASSERT_EQ(pipeline->issue(inst.release(), *wf), VmAccessOutcome::Complete); EXPECT_EQ(read_memory(address, dwords), in_range ? 7u : 6u); - EXPECT_EQ(read_sgprs(4, dwords), !glc ? 7u : (in_range ? 6u : 0u)); + EXPECT_EQ(read_sgprs(4, dwords), !glc ? data : (in_range ? 6u : 0u)); EXPECT_TRUE(wf->wait_counters().empty()); if (backing && !in_range) { EXPECT_EQ(backing->load_attempts, 0u);