[rocjitsu][amdisa] Model s_buffer_atomic_* instructions

emulation/rocjitsu/docs/vm-design.md

@@ -563,13 +563,21 @@ SC0/GLC is set. The L1 line is invalidated after the atomic to prevent
 stale reads. Supported integer atomics: swap, cmpswap, add, sub,
 smin/umin, smax/umax, and, or, xor, inc, dec.
 
-**Scalar atomics** (`s_atomic_*` and `s_atomic_*_x2`, CDNA1–4) bypass the
-L1 scalar cache and perform read-modify-write at L2; the atomic boundary's
-coherence epoch makes stale K$ lines refetch. The old value is returned to
-SDATA when GLC is set. Supported: swap, cmpswap, add, sub, smin/umin,
-smax/umax, and, or, xor, inc, dec. Compare-swap reads the source and then
-the compare value from SDATA and returns only the old value, to the low half.
-`s_buffer_atomic_*` are not yet implemented.
+**Scalar atomics** (`s_atomic_*`, `s_buffer_atomic_*` and their `_x2` forms,
+CDNA1–4) bypass the L1 scalar cache and perform read-modify-write at L2; the
+atomic boundary's coherence epoch makes stale K$ lines refetch. The old value
+is returned to SDATA when GLC is set. Supported: swap, cmpswap, add, sub,
+smin/umin, smax/umax, and, or, xor, inc, dec. Compare-swap reads the source
+and then the compare value from SDATA and returns only the old value, to the
+low half. A buffer atomic's address is the V# base plus the offset; stride
+only scales the bound. A buffer atomic whose element is not wholly in range
+performs no access and returns zero when GLC is set. A misaligned `_x2` buffer
+atomic faults even when out of range. The CDNA ISA manuals do not describe
+these bounds and fault-order rules. They are observed on gfx90a with
+`s_buffer_atomic_inc[_x2]`, and the fault order also with
+`s_buffer_atomic_dec_x2`. The other operations and CDNA targets are modeled
+the same way by extrapolation, including `s_buffer_atomic_cmpswap_x2`, whose
+bound covers its 8-byte memory element although SDATA holds 16 bytes.
 
 **Cache management instructions:**
 

emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py

@@ -8378,11 +8378,6 @@ def _gen_smem_store(
         return '\n'.join(L)
 
     def _gen_smem_atomic(self, sem: InstructionSemantics) -> str:
-        # This emitter computes a global address; buffer forms need V# addressing.
-        if sem.name.upper().startswith('S_BUFFER_ATOMIC_'):
-            raise ValueError(
-                f'{sem.name}: smem_atomic supports only global S_ATOMIC_* addressing'
-            )
         # Semantics has no op-less smem_atomic fallback, so a missing op is a
         # derivation bug rather than an unimplemented variant.
         op_enum = self._ATOMIC_OP_ENUM.get(sem.operation)
@@ -8417,7 +8412,7 @@ def _gen_smem_atomic(self, sem: InstructionSemantics) -> str:
         self._append_wait_counter_type(L, sem)
         L.extend(
             [
-                '  auto address = smem_calculate_address(inst_, wf);',
+                '  auto address = smem_calculate_address(inst_, wf, d.get());',
                 '  if (!address) return;',
                 '  d->addr = *address;',
                 '  set_data(std::move(d));',
@@ -11545,7 +11540,7 @@ def gen_insts(self) -> None:
                                 f'modifiers.block_data = &{self.isa_spec.profile.flat_store_src_field};'
                             )
                         if is_smem and inst.name.startswith(
-                            ('S_BUFFER_LOAD_', 'S_BUFFER_STORE_')
+                            ('S_BUFFER_LOAD_', 'S_BUFFER_STORE_', 'S_BUFFER_ATOMIC_')
                         ):
                             words = (
                                 2

emulation/rocjitsu/lib/python/amdisa/semantics.py

@@ -1877,23 +1877,24 @@ def _derive_vop3p(name: str) -> InstructionSemantics | None:
     'U16': (2, False),
 }
 
-# S_ATOMIC_* operations the CDNA ISAs define and the scalar pipeline supports.
-# Other flat atomic operations stay 'nop' even when the flat tables know them.
+# S_ATOMIC_* and S_BUFFER_ATOMIC_* operations the CDNA ISAs define and the
+# scalar pipeline supports, spelled as in the mnemonic without _X2. Any other
+# scalar atomic derives nop, even when the flat tables know its operation.
 _SMEM_ATOMIC_OPS = frozenset(
     {
-        'swap',
-        'cmpswap',
-        'add',
-        'sub',
-        'smin',
-        'umin',
-        'smax',
-        'umax',
-        'and',
-        'or',
-        'xor',
-        'inc',
-        'dec',
+        'SWAP',
+        'CMPSWAP',
+        'ADD',
+        'SUB',
+        'SMIN',
+        'UMIN',
+        'SMAX',
+        'UMAX',
+        'AND',
+        'OR',
+        'XOR',
+        'INC',
+        'DEC',
     }
 )
 
@@ -1928,23 +1929,20 @@ def _derive_smem(name: str) -> InstructionSemantics | None:
     if 'DCACHE' in upper or upper in ('S_ATC_PROBE', 'S_ATC_PROBE_BUFFER'):
         return InstructionSemantics(name, 'nop')
 
-    if '_ATOMIC_' in upper:
-        if upper.startswith('S_ATOMIC_'):
-            suffix = upper[len('S_ATOMIC_') :]
-            is_x2 = suffix.endswith('_X2')
-            if is_x2:
-                suffix = suffix[:-3]
-            info = _derive_flat_atomic_info(suffix, is_x2)
-            if info and info[0] in _SMEM_ATOMIC_OPS:
-                op, elem_size, data_dw_actual = info
-                return InstructionSemantics(
-                    name,
-                    'smem_atomic',
-                    operation=op,
-                    elem_size=elem_size,
-                    num_elems=data_dw_actual,
-                )
-        # Scalar buffer atomics are not currently simulated.
+    if upper.startswith(('S_ATOMIC_', 'S_BUFFER_ATOMIC_')):
+        suffix = upper.partition('_ATOMIC_')[2]
+        is_x2 = suffix.endswith('_X2')
+        if is_x2:
+            suffix = suffix[:-3]
+        if suffix in _SMEM_ATOMIC_OPS:
+            op, elem_size, data_dw_actual = _derive_flat_atomic_info(suffix, is_x2)
+            return InstructionSemantics(
+                name,
+                'smem_atomic',
+                operation=op,
+                elem_size=elem_size,
+                num_elems=data_dw_actual,
+            )
         return InstructionSemantics(name, 'nop')
 
     is_store = '_STORE_' in upper or '_SCRATCH_STORE_' in upper

emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py

@@ -2377,11 +2377,12 @@ def test_gfx1250_prefetch_hints_are_true_nop(self, name):
 
 # Compare-swap carries its compare value after the source, doubling its data.
 _IMPLEMENTED_SCALAR_ATOMICS = {
-    f'S_ATOMIC_{op}{width}': (
+    f'{prefix}{op}{width}': (
         op.lower(),
         elem_size,
         2 * num_elems if op == 'CMPSWAP' else num_elems,
     )
+    for prefix in ('S_ATOMIC_', 'S_BUFFER_ATOMIC_')
     for op in _SCALAR_ATOMIC_OPS
     for width, elem_size, num_elems in (('', 4, 1), ('_X2', 8, 2))
 }
@@ -2398,23 +2399,9 @@ def test_implemented_scalar_atomics_derive_operation_and_widths(self, name):
         assert sem.elem_size == elem_size
         assert sem.num_elems == num_elems
 
-    @pytest.mark.parametrize(
-        'name',
-        [
-            f'{prefix}{op}{width}'
-            for prefix in ('S_ATOMIC_', 'S_BUFFER_ATOMIC_')
-            for op in _SCALAR_ATOMIC_OPS
-            for width in ('', '_X2')
-            if f'{prefix}{op}{width}' not in _IMPLEMENTED_SCALAR_ATOMICS
-        ],
-    )
-    def test_unimplemented_scalar_atomics_stay_nop(self, name):
-        sem = derive_semantics(name, 'ENC_SMEM')
-        assert sem is not None
-        assert sem.semantic_class == 'nop'
-
-    # No CDNA ISA defines these. The flat atomic tables know their operations,
-    # so they pin that scalar support comes from the allowlist, not that table.
+    # No CDNA ISA defines these. They pin that scalar support comes from the
+    # allowlist: the flat atomic tables know most of these operations, SUB_F32
+    # strips to an allowlisted SUB, and ADD_U16 ends in a narrow load suffix.
     @pytest.mark.parametrize(
         'name',
         [
@@ -2423,6 +2410,10 @@ def test_unimplemented_scalar_atomics_stay_nop(self, name):
             'S_ATOMIC_PK_ADD_F16',
             'S_ATOMIC_CSUB',
             'S_ATOMIC_COND_SUB_U32',
+            'S_BUFFER_ATOMIC_ADD_F32',
+            'S_BUFFER_ATOMIC_FMIN',
+            'S_BUFFER_ATOMIC_SUB_F32',
+            'S_BUFFER_ATOMIC_ADD_U16',
         ],
     )
     def test_scalar_atomic_operations_outside_allowlist_stay_nop(self, name):

emulation/rocjitsu/lib/python/amdisa/tests/test_smem_atomic_contract.py

@@ -31,8 +31,11 @@ def _codegen(arch_name, profile):
         ('cdna4', Cdna4Profile()),
     ],
 )
-@pytest.mark.parametrize('name', ['S_ATOMIC_DEC', 's_atomic_dec'])
-def test_smem_atomic_accepts_derived_global_decrement(arch_name, profile, name):
+@pytest.mark.parametrize(
+    'name',
+    ['S_ATOMIC_DEC', 's_atomic_dec', 'S_BUFFER_ATOMIC_DEC', 's_buffer_atomic_dec'],
+)
+def test_smem_atomic_accepts_derived_decrement(arch_name, profile, name):
     sem = derive_semantics(name, 'ENC_SMEM')
     assert sem is not None
     assert sem.semantic_class == 'smem_atomic'
@@ -42,7 +45,8 @@ def test_smem_atomic_accepts_derived_global_decrement(arch_name, profile, name):
     assert 'd->atomic_op = amdgpu::AtomicOp::DEC;' in body
     assert 'd->elem_size = 4;' in body
     assert 'd->num_dwords = 1;' in body
-    assert 'smem_calculate_address(inst_, wf);' in body
+    # The state receives the buffer bounds mask; global atomics leave it full.
+    assert 'smem_calculate_address(inst_, wf, d.get());' in body
 
 
 @pytest.mark.parametrize(
@@ -85,11 +89,6 @@ def test_smem_atomic_payload_and_return_widths_follow_semantics(
 @pytest.mark.parametrize(
     ('changes', 'message'),
     [
-        pytest.param(
-            {'name': 'S_BUFFER_ATOMIC_DEC'},
-            'supports only global S_ATOMIC_',
-            id='buffer-mnemonic',
-        ),
         pytest.param(
             {'operation': 'unknown'},
             'unsupported smem_atomic operation',