From 177c5e7937ac05982e870b6d6b9cc454044e9793 Mon Sep 17 00:00:00 2001 From: Sambhav Jain Date: Fri, 28 Aug 2026 18:42:55 +0000 Subject: [PATCH 01/16] [rocjitsu] Saturate clamped VOP3 integer arithmetic VOP3 integer add and subtract semantic lowering ignored the encoded CLAMP bit and narrowed wrapped results directly. On gfx1250 this made v_sub_nc_u32 v4, v4, 4 clamp turn an underflow into 0xfffffffd instead of zero, corrupting exponent normalization in FP8 conversion (ROCm/rocm-systems#10760). Route signed and unsigned 16- and 32-bit VOP3 add/sub operations through shared helpers that widen, saturate to the destination type, and then narrow. Preserve modulo arithmetic when CLAMP is clear, and make the generic SIMD fast path fall back to the authoritative scalar body for clamped operations. Regenerate the affected CDNA and RDNA execution sources. Add generator coverage plus a gfx1250 execution regression using the exact reported instruction encoding. Exercise signed and unsigned overflow and underflow boundaries at both widths, unclamped wrapping, and normal versus forced-scalar execution. Verified with the full amdisa Python suite (1901 passed, 46 skipped), focused native execution/SIMD tests, a full build, and the original PyTorch FP8 reproducer. Signed-off-by: Sambhav Jain --- .../lib/python/amdisa/codegen/_generator.py | 9 ++ .../amdisa/codegen/execute/sema_lower.py | 22 ++++ .../amdisa/codegen/execute/simd_codegen.py | 6 +- .../python/amdisa/tests/test_sema_derive.py | 25 ++++ .../arch/amdgpu/generated/cdna1/vop3_exec.cpp | 52 ++++---- .../arch/amdgpu/generated/cdna2/vop3_exec.cpp | 52 ++++---- .../arch/amdgpu/generated/cdna3/vop3_exec.cpp | 52 ++++---- .../arch/amdgpu/generated/cdna4/vop3_exec.cpp | 52 ++++---- .../amdgpu/generated/cdna5/vop3_exec_alu.cpp | 80 ++++++------ .../arch/amdgpu/generated/rdna3/vop3_exec.cpp | 80 ++++++------ .../amdgpu/generated/rdna3_5/vop3_exec.cpp | 80 ++++++------ .../arch/amdgpu/generated/rdna4/vop3_exec.cpp | 80 ++++++------ .../amdgpu/generated/shared/execute_shared.h | 114 +++++++++++------- .../isa/arch/amdgpu/shared/simd_glue.h | 54 ++++++++- .../rocjitsu/tests/cdna5_execution_test.cpp | 66 ++++++++++ 15 files changed, 536 insertions(+), 288 deletions(-) diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py index 232a2bbcf4d2..b8e45f95874e 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py @@ -6148,6 +6148,15 @@ def _semantic_reg_class(opnd: Operand) -> RegClass: mode_sensitive_f16_dst=not cls.startswith('scalar_'), mask_result_writer=result_writer, ) + inst_fields = getattr(self, '_current_inst_fields', set()) + if ( + is_vop3 + and cls == 'vector_binop' + and op in ('add', 'sub', 'subrev', 'rsub') + and dtype in ('i16', 'u16', 'i32', 'u32') + and 'clamp' in inst_fields + ): + lctx.integer_saturation_dtype = dtype if cls == 'vector_cmp': # V_CMP writes a fresh wave mask initialized to zero, so false # lanes can remain clear without emitting redundant bit clears. diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py index 8aa702220893..4905760b6e1a 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py @@ -128,6 +128,7 @@ class LoweringContext: vector_sgpr_once: bool = False clear_false_lane_mask_writes: bool = True mode_sensitive_f16_dst: bool = True + integer_saturation_dtype: str | None = None _INFIX_OPS: dict[SemaNodeKind, str] = { @@ -537,6 +538,12 @@ def _lower_expr(node: SemaNode, ctx: LoweringContext) -> str: if kind == SemaNodeKind.ID: return _lower_id(node, ctx) + if ( + kind in (SemaNodeKind.ADD, SemaNodeKind.SUB) + and ctx.integer_saturation_dtype is not None + ): + return _lower_saturating_integer_binary(node, ctx) + if kind in _INFIX_OPS: if (expr := _lower_less_greater_once(node, ctx)) is not None: return expr @@ -727,6 +734,21 @@ def _lower_expr(node: SemaNode, ctx: LoweringContext) -> str: raise ValueError(f'Unhandled SemaNodeKind in lowering: {kind.name}') +def _lower_saturating_integer_binary(node: SemaNode, ctx: LoweringContext) -> str: + """Lower VOP3 integer add/sub with optional CLAMP saturation.""" + dtype = ctx.integer_saturation_dtype + if dtype not in ('i16', 'u16', 'i32', 'u32'): + raise ValueError(f'Unsupported integer saturation type: {dtype}') + + lhs = _lower_expr(node.children[0], replace(ctx, integer_saturation_dtype=None)) + rhs = _lower_expr(node.children[1], replace(ctx, integer_saturation_dtype=None)) + operation = 'add' if node.kind == SemaNodeKind.ADD else 'sub' + cpp_type = f'{"int" if dtype.startswith("i") else "uint"}{dtype[1:]}_t' + return ( + f'amdgpu::vop3_integer_{operation}<{cpp_type}>(' f'{lhs}, {rhs}, inst_.clamp)' + ) + + def _lower_lit(node: SemaNode) -> str: """Lower a literal value with appropriate C++ suffix.""" val = node.lit_value or '0' diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/simd_codegen.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/simd_codegen.py index 7391f850187e..5dde1b5ee1a2 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/simd_codegen.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/simd_codegen.py @@ -1735,10 +1735,12 @@ def _build_simd_vopc_vop3_f64() -> dict[str, str]: # # 16-bit forms compute a 32-bit add/sub and mask the low 16 bits, matching the # scalar body's `uint32_t(uint16_t(int16_t(low16(a)+low16(b))))` (or the u16 -# variant) — both reduce to `(a + b) & 0xFFFFu` / `(a - b) & 0xFFFFu` because +# variant) — both reduce to `(a + b) & 0xFFFFu` / `(a - b) & 0xFFFFu` when +# CLAMP is clear because # unsigned 32-bit wrap-around at the low 16 bits is identical to signed/unsigned # 16-bit wrap. 32-bit forms use the wrap-around add/sub on uint32 lanes; -# signed-vs-unsigned wraps the same way. +# signed-vs-unsigned wraps the same way. The shared glue falls back to scalar +# execution when CLAMP requests saturating arithmetic. SIMD_VOP3_BINARY_INT_EXTRA: dict[str, tuple[str, str]] = { # v_bcnt_u32_b32: VOP3-only (no VOP1 twin). D = CountOneBits(S0) + S1 -- the # second source is an accumulator, so this is binary, not unary. Keep the diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py index e963a575c0b5..5eab503368fc 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py @@ -1579,6 +1579,31 @@ def test_signed_add_sub_use_unsigned_operands_to_avoid_ub(self): not in cpp ) + @pytest.mark.parametrize( + 'op,dtype,expected', + [ + ('add', 'u32', 'vop3_integer_add'), + ('sub', 'u32', 'vop3_integer_sub'), + ('subrev', 'u32', 'vop3_integer_sub'), + ('add', 'i32', 'vop3_integer_add'), + ('sub', 'i32', 'vop3_integer_sub'), + ('add', 'u16', 'vop3_integer_add'), + ('sub', 'i16', 'vop3_integer_sub'), + ], + ) + def test_vop3_integer_clamp_saturates_before_narrowing(self, op, dtype, expected): + sem = _FakeSem(f'V_{op.upper()}_{dtype.upper()}', 'vector_binop', op, dtype) + block = derive_sema_block(sem) + ctx = LoweringContext( + exec_model=block.pragma, + integer_saturation_dtype=dtype, + ) + + cpp = lower_sema_block(block, ctx) + + assert 'inst_.clamp' in cpp + assert expected in cpp + def test_lshlrev(self): sem = _FakeSem('V_LSHLREV_B32', 'vector_binop', 'lshlrev', 'b32') block = derive_sema_block(sem) diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3_exec.cpp index 2e1d1987ea7b..46a0696c9909 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3_exec.cpp @@ -1438,11 +1438,13 @@ void VAddU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1455,11 +1457,13 @@ void VSubU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1472,11 +1476,13 @@ void VSubrevU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -3034,10 +3040,10 @@ void VAddI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -3050,10 +3056,10 @@ void VSubI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3_exec.cpp index b7ad72ed9d1e..282c6be18758 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3_exec.cpp @@ -1430,11 +1430,13 @@ void VAddU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1447,11 +1449,13 @@ void VSubU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1464,11 +1468,13 @@ void VSubrevU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -3006,10 +3012,10 @@ void VAddI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -3022,10 +3028,10 @@ void VSubI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3_exec.cpp index be274c1aab61..9bfb1ca7d05a 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3_exec.cpp @@ -1505,11 +1505,13 @@ void VAddU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1522,11 +1524,13 @@ void VSubU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1539,11 +1543,13 @@ void VSubrevU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -3079,10 +3085,10 @@ void VAddI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -3095,10 +3101,10 @@ void VSubI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3_exec.cpp index 97a6b5ac5ad7..c9e6a2d412b3 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3_exec.cpp @@ -1565,11 +1565,13 @@ void VAddU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1582,11 +1584,13 @@ void VSubU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1599,11 +1603,13 @@ void VSubrevU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -7450,10 +7456,10 @@ void VAddI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -7466,10 +7472,10 @@ void VSubI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_alu.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_alu.cpp index 8f510f3e4c39..b5786f33c56c 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_alu.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_alu.cpp @@ -5718,11 +5718,13 @@ void VAddNcU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -5752,11 +5754,13 @@ RJ_NOINLINE void VAddNcU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -5774,11 +5778,13 @@ void VSubNcU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -5808,11 +5814,13 @@ RJ_NOINLINE void VSubNcU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -6120,10 +6128,10 @@ void VAddNcI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -6153,10 +6161,10 @@ RJ_NOINLINE void VAddNcI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -6174,10 +6182,10 @@ void VSubNcI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -6207,10 +6215,10 @@ RJ_NOINLINE void VSubNcI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop3_exec.cpp index a0575d034104..4a26b3c16d07 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop3_exec.cpp @@ -9199,11 +9199,13 @@ void VAddNcU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9233,11 +9235,13 @@ RJ_NOINLINE void VAddNcU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9255,11 +9259,13 @@ void VSubNcU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9289,11 +9295,13 @@ RJ_NOINLINE void VSubNcU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9701,10 +9709,10 @@ void VAddNcI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9734,10 +9742,10 @@ RJ_NOINLINE void VAddNcI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9755,10 +9763,10 @@ void VSubNcI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9788,10 +9796,10 @@ RJ_NOINLINE void VSubNcI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop3_exec.cpp index c7efd85fc66f..cb6c9167f5b6 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop3_exec.cpp @@ -9199,11 +9199,13 @@ void VAddNcU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9233,11 +9235,13 @@ RJ_NOINLINE void VAddNcU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9255,11 +9259,13 @@ void VSubNcU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9289,11 +9295,13 @@ RJ_NOINLINE void VSubNcU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9701,10 +9709,10 @@ void VAddNcI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9734,10 +9742,10 @@ RJ_NOINLINE void VAddNcI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9755,10 +9763,10 @@ void VSubNcI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9788,10 +9796,10 @@ RJ_NOINLINE void VSubNcI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3_exec.cpp index 1dff41bf50f6..9053343509b8 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3_exec.cpp @@ -10212,11 +10212,13 @@ void VAddNcU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -10246,11 +10248,13 @@ RJ_NOINLINE void VAddNcU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -10268,11 +10272,13 @@ void VSubNcU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -10302,11 +10308,13 @@ RJ_NOINLINE void VSubNcU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -10714,10 +10722,10 @@ void VAddNcI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -10747,10 +10755,10 @@ RJ_NOINLINE void VAddNcI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -10768,10 +10776,10 @@ void VSubNcI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -10801,10 +10809,10 @@ RJ_NOINLINE void VSubNcI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h index 007db517045e..154454df9190 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h @@ -3217,9 +3217,11 @@ inline void execute_v_add_i32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) + - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_add(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + inst.inst_.clamp)); } } @@ -3246,10 +3248,12 @@ inline void execute_v_add_nc_i16_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - static_cast(static_cast(static_cast( - (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) + - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)))))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), inst.inst_.clamp))))); } } @@ -3260,9 +3264,11 @@ inline void execute_v_add_nc_i32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) + - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_add(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + inst.inst_.clamp)); } } @@ -3274,9 +3280,10 @@ inline void execute_v_add_nc_u16_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse continue; sdwa::write_lane( inst, wf, inst.vdst, lane, - static_cast(static_cast( - (static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)) + - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)))))); + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.clamp)))); } } @@ -3300,9 +3307,11 @@ inline void execute_v_add_nc_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) + - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_add(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + inst.inst_.clamp)); } } @@ -3341,9 +3350,11 @@ inline void execute_v_add_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) + - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_add(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + inst.inst_.clamp)); } } @@ -19222,9 +19233,11 @@ inline void execute_v_sub_i32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) - - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_sub(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + inst.inst_.clamp)); } } @@ -19234,10 +19247,12 @@ inline void execute_v_sub_nc_i16_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - static_cast(static_cast(static_cast( - (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) - - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)))))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), inst.inst_.clamp))))); } } @@ -19248,9 +19263,11 @@ inline void execute_v_sub_nc_i32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) - - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_sub(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + inst.inst_.clamp)); } } @@ -19262,9 +19279,10 @@ inline void execute_v_sub_nc_u16_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse continue; sdwa::write_lane( inst, wf, inst.vdst, lane, - static_cast(static_cast( - (static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)) - - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)))))); + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.clamp)))); } } @@ -19288,9 +19306,11 @@ inline void execute_v_sub_nc_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) - - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_sub(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + inst.inst_.clamp)); } } @@ -19329,9 +19349,11 @@ inline void execute_v_sub_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) - - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_sub(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + inst.inst_.clamp)); } } @@ -19733,9 +19755,11 @@ inline void execute_v_subrev_nc_u32_vop3([[maybe_unused]] Inst &inst, for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - (amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane) - - amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_sub(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + inst.inst_.clamp)); } } @@ -19774,9 +19798,11 @@ inline void execute_v_subrev_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - (amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane) - - amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_sub(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + inst.inst_.clamp)); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h index 7180bb28d1ef..bcfd276a730d 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h @@ -24,6 +24,7 @@ #include "simdojo/components/vector_reg.h" #include "util/simd.h" +#include #include #include #include @@ -94,6 +95,48 @@ template inline T clamp_floating_result(T value, const Wavefront &w return clamp_floating_result(value, floating_clamp_nan_to_zero(wf)); } +/// @brief Execute VOP3 integer addition, saturating when CLAMP is set. +template inline uint32_t vop3_integer_add(uint32_t lhs, uint32_t rhs, bool clamp) { + static_assert(std::is_integral_v && (sizeof(T) == 2 || sizeof(T) == 4)); + using U = std::make_unsigned_t; + const U lhs_bits = static_cast(lhs); + const U rhs_bits = static_cast(rhs); + if (!clamp) + return static_cast(static_cast(lhs_bits + rhs_bits)); + if constexpr (std::is_signed_v) { + using Wide = std::conditional_t; + const Wide wide = static_cast(std::bit_cast(lhs_bits)) + + static_cast(std::bit_cast(rhs_bits)); + const Wide saturated = std::clamp(wide, static_cast(std::numeric_limits::min()), + static_cast(std::numeric_limits::max())); + return static_cast(static_cast(saturated)); + } else { + using Wide = std::conditional_t; + const Wide wide = static_cast(lhs_bits) + static_cast(rhs_bits); + return static_cast(std::min(wide, static_cast(std::numeric_limits::max()))); + } +} + +/// @brief Execute VOP3 integer subtraction, saturating when CLAMP is set. +template inline uint32_t vop3_integer_sub(uint32_t lhs, uint32_t rhs, bool clamp) { + static_assert(std::is_integral_v && (sizeof(T) == 2 || sizeof(T) == 4)); + using U = std::make_unsigned_t; + const U lhs_bits = static_cast(lhs); + const U rhs_bits = static_cast(rhs); + if (!clamp) + return static_cast(static_cast(lhs_bits - rhs_bits)); + if constexpr (std::is_signed_v) { + using Wide = std::conditional_t; + const Wide wide = static_cast(std::bit_cast(lhs_bits)) - + static_cast(std::bit_cast(rhs_bits)); + const Wide saturated = std::clamp(wide, static_cast(std::numeric_limits::min()), + static_cast(std::numeric_limits::max())); + return static_cast(static_cast(saturated)); + } else { + return lhs_bits < rhs_bits ? 0u : static_cast(lhs_bits - rhs_bits); + } +} + /// @brief Return whether V_DOT4 integer instructions honor the encoded CLAMP bit. /// @details GFX12 (RDNA4) and the CDNA5-backed gfx1250 profile explicitly ignore CLAMP /// for the integer DOT4 forms; earlier CDNA/RDNA profiles retain the ordinary @@ -1675,15 +1718,14 @@ template /// VOP3 integer/bitwise binary SIMD fast path. Same shape as /// try_execute_binary_vop2_simd but reads the VOP3 operands `src0`/`src1` /// (instead of `src0`/`vsrc1`). The generated integer/bitwise VOP3 bodies apply -/// no source/result modifiers (abs/neg/omod are float-only; clamp on an integer -/// op means saturate, which these wrap-around/bitwise twins do not request), so -/// the plain op is bit-identical to the scalar body on every input. T is a -/// 32-bit integer lane type. +/// no source/result modifiers other than integer CLAMP. Saturating integer +/// arithmetic falls back to the scalar body; the plain op is bit-identical to +/// that body when CLAMP is clear. T is a 32-bit integer lane type. template requires(util::has_stdx_simd) [[nodiscard]] inline bool try_execute_binary_vop3_simd(Inst &inst, Wavefront &wf, BinOp bin_op) { - if (simd_force_scalar() || !sdwa::supports_direct_simd_store(inst) || !inst.src0.simd_capable() || - !inst.src1.simd_capable() || !inst.vdst.simd_capable()) + if (inst.inst_.clamp != 0u || simd_force_scalar() || !sdwa::supports_direct_simd_store(inst) || + !inst.src0.simd_capable() || !inst.src1.simd_capable() || !inst.vdst.simd_capable()) return false; constexpr std::size_t W = util::native_width_v; const uint64_t chunk_full = util::mask(static_cast(W)); diff --git a/emulation/rocjitsu/tests/cdna5_execution_test.cpp b/emulation/rocjitsu/tests/cdna5_execution_test.cpp index d409f6bd8b1e..8b7bf7f91db4 100644 --- a/emulation/rocjitsu/tests/cdna5_execution_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_execution_test.cpp @@ -156,6 +156,72 @@ TEST(Gfx1250ExecutionTest, GenericF16OmodStaysActiveAndFinalizesWithIeeeDenormMo } } +TEST(Gfx1250ExecutionTest, Vop3IntegerClampSaturatesBeforeNarrowing) { + ForceScalarGuard guard; + for (uint32_t scalar = 0; scalar < 2; ++scalar) { + util::set_force_scalar_for_testing(scalar != 0); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + const uint32_t base = wf->vgpr_alloc().base; + + // Exact instruction from ROCm/rocm-systems#10760: + // v_sub_nc_u32 v4, v4, 4 clamp + cu->write_vgpr(base + 4, 0, 1u); + constexpr std::array kIssueInstruction = {0xd5268004u, 0x02010904u}; + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA5); + ASSERT_NE(decoder, nullptr); + std::unique_ptr issue_inst(decode_valid(*decoder, kIssueInstruction.data())); + ASSERT_NE(issue_inst, nullptr); + EXPECT_EQ(issue_inst->mnemonic(), "v_sub_nc_u32"); + issue_inst->execute(*issue_inst, wf); + EXPECT_EQ(cu->read_vgpr(base + 4, 0), 0u) << "scalar " << scalar; + + struct TestCase { + uint16_t opcode; + uint32_t lhs; + uint32_t rhs; + uint32_t expected; + }; + constexpr std::array kBoundaryCases = {{ + {cdna5::kVAddNcU32Vop3, UINT32_MAX, 1u, UINT32_MAX}, + {cdna5::kVSubNcU32Vop3, 7u, 3u, 4u}, + {cdna5::kVAddNcI32Vop3, static_cast(INT32_MAX), 1u, + static_cast(INT32_MAX)}, + {cdna5::kVSubNcI32Vop3, static_cast(INT32_MIN), 1u, + static_cast(INT32_MIN)}, + {cdna5::kVAddNcU16Vop3, UINT16_MAX, 1u, UINT16_MAX}, + {cdna5::kVSubNcU16Vop3, 0u, 1u, 0u}, + {cdna5::kVAddNcI16Vop3, static_cast(INT16_MAX), 1u, + static_cast(INT16_MAX)}, + {cdna5::kVSubNcI16Vop3, static_cast(INT16_MIN), 1u, + static_cast(INT16_MIN)}, + }}; + for (const auto &test : kBoundaryCases) { + cu->write_vgpr(base + 0, 0, test.lhs); + cu->write_vgpr(base + 1, 0, test.rhs); + cu->write_vgpr(base + 2, 0, 0u); + const auto words = + cdna5::build_vop3(test.opcode, {.vdst = 2, .clamp = 1, .src0 = 256, .src1 = 257}); + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + inst->execute(*inst, wf); + EXPECT_EQ(cu->read_vgpr(base + 2, 0), test.expected) << "scalar " << scalar; + } + + // CLAMP=0 retains the architectural modulo result. + cu->write_vgpr(base + 0, 0, UINT32_MAX); + cu->write_vgpr(base + 1, 0, 1u); + const auto wrap_words = + cdna5::build_vop3(cdna5::kVAddNcU32Vop3, {.vdst = 2, .src0 = 256, .src1 = 257}); + cdna5::VAddNcU32Vop3 wrap_inst(wrap_words.data()); + wrap_inst.execute_impl(*wf); + EXPECT_EQ(cu->read_vgpr(base + 2, 0), 0u) << "scalar " << scalar; + } +} + TEST(FpModePolicyTest, F64HelpersRestoreAmbientHostEnvironment) { HostFenvGuard environment_guard; ASSERT_EQ(std::fesetround(FE_DOWNWARD), 0); From 894bf7797ad04f6271dbee3ead771a4522eb048e Mon Sep 17 00:00:00 2001 From: Sambhav Jain Date: Sat, 29 Aug 2026 00:53:08 +0000 Subject: [PATCH 02/16] [rocjitsu] Cover remaining clamped integer arithmetic Extend CLAMP saturation to packed I16/U16 add/sub and VOP3 carry/borrow forms while preserving full-width carry results. Gate clamped SIMD paths to scalar and add generator and execution regressions for selectors, reverse subtract, carry outputs, and signed/unsigned boundaries. Signed-off-by: Sambhav Jain --- .../lib/python/amdisa/codegen/_generator.py | 11 ++- .../python/amdisa/codegen/execute/packed.py | 24 ++++- .../amdisa/codegen/execute/sema_lower.py | 25 +++++ .../tests/test_generator_profile_gates.py | 41 +++++++++ .../python/amdisa/tests/test_sema_derive.py | 50 ++++++++++ .../amdgpu/generated/cdna5/vop3p_exec.cpp | 24 +++-- .../amdgpu/generated/rdna4/vop3p_exec.cpp | 24 +++-- .../amdgpu/generated/shared/execute_shared.h | 82 +++++++++-------- .../isa/arch/amdgpu/shared/simd_glue.h | 14 +-- .../rocjitsu/tests/cdna5_execution_test.cpp | 92 ++++++++++++++++++- 10 files changed, 319 insertions(+), 68 deletions(-) diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py index b8e45f95874e..bc39f5ef421f 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py @@ -6151,10 +6151,15 @@ def _semantic_reg_class(opnd: Operand) -> RegClass: inst_fields = getattr(self, '_current_inst_fields', set()) if ( is_vop3 - and cls == 'vector_binop' - and op in ('add', 'sub', 'subrev', 'rsub') - and dtype in ('i16', 'u16', 'i32', 'u32') and 'clamp' in inst_fields + and ( + ( + cls == 'vector_binop' + and op in ('add', 'sub', 'subrev', 'rsub') + and dtype in ('i16', 'u16', 'i32', 'u32') + ) + or (cls == 'vector_add_co' and dtype == 'u32') + ) ): lctx.integer_saturation_dtype = dtype if cls == 'vector_cmp': diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py index 97bb39d5e866..e17aa9d3f43d 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py @@ -179,8 +179,16 @@ def gen_pk_binop( ), } lo_expr, hi_expr = i_op_map[op] - L.append(f' uint16_t rlo = static_cast({lo_expr});') - L.append(f' uint16_t rhi = static_cast({hi_expr});') + if op in ('add', 'sub'): + L.append( + f' uint16_t rlo = static_cast(amdgpu::vop3_integer_{op}(a_lo, b_lo, inst_.clamp));' + ) + L.append( + f' uint16_t rhi = static_cast(amdgpu::vop3_integer_{op}(a_hi, b_hi, inst_.clamp));' + ) + else: + L.append(f' uint16_t rlo = static_cast({lo_expr});') + L.append(f' uint16_t rhi = static_cast({hi_expr});') L.append( f' amdgpu::RegisterAccess(wf).write_lane({d}, lane, static_cast(rlo) | (static_cast(rhi) << 16));' ) @@ -216,8 +224,16 @@ def gen_pk_binop( ), } lo_expr, hi_expr = u_op_map[op] - L.append(f' uint16_t rlo = static_cast({lo_expr});') - L.append(f' uint16_t rhi = static_cast({hi_expr});') + if op in ('add', 'sub'): + L.append( + f' uint16_t rlo = static_cast(amdgpu::vop3_integer_{op}(a_lo, b_lo, inst_.clamp));' + ) + L.append( + f' uint16_t rhi = static_cast(amdgpu::vop3_integer_{op}(a_hi, b_hi, inst_.clamp));' + ) + else: + L.append(f' uint16_t rlo = static_cast({lo_expr});') + L.append(f' uint16_t rhi = static_cast({hi_expr});') L.append( f' amdgpu::RegisterAccess(wf).write_lane({d}, lane, static_cast(rlo) | (static_cast(rhi) << 16));' ) diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py index 4905760b6e1a..9e8039890764 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py @@ -1751,6 +1751,31 @@ def _lower_call(node: SemaNode, ctx: LoweringContext) -> str: args = [_lower_expr(c, ctx) for c in node.children[1:]] args_str = ', '.join(args) + if ctx.integer_saturation_dtype == 'u32' and callee in ( + 'add_co', + 'sub_co', + 'addc_co', + 'subbc_co', + ): + if callee in ('add_co', 'addc_co'): + terms = ' + '.join(f'static_cast({arg})' for arg in args) + return ( + f'[&]() {{ uint64_t w = {terms};' + ' if (w > 0xFFFFFFFFULL) vcc |= (1ULL << lane);' + ' else vcc &= ~(1ULL << lane);' + ' return inst_.clamp && w > 0xFFFFFFFFULL ? UINT32_MAX' + ' : static_cast(w); }()' + ) + minuend, subtrahend, *borrow = args + borrow_expr = f' + static_cast({borrow[0]})' if borrow else '' + return ( + f'[&]() {{ uint64_t a = static_cast({minuend}),' + f' b = static_cast({subtrahend}){borrow_expr};' + ' if (a < b) vcc |= (1ULL << lane);' + ' else vcc &= ~(1ULL << lane);' + ' return inst_.clamp && a < b ? 0u : static_cast(a - b); }()' + ) + pseudo_scalar_operations = { 'exp2': 'EXP2', 'log2': 'LOG2', diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py index dd7e2a875bc9..4184c70cb1ee 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py @@ -4077,6 +4077,47 @@ def test_gfx1250_generated_vop3_add_f16_applies_dpp( assert 'src0.clear_delegate();' not in body +def test_gfx1250_generator_wires_integer_clamp_only_from_encoded_vop3_field(): + isa_xml = _mrisa_dir() / 'amdgpu_isa_cdna5.xml' + parser = Parser(str(isa_xml), Cdna5Profile()) + spec = parser.parse() + semantics = derive_all_semantics(spec) + generator = CodeGenerator(spec, '', semantics) + + def generated_body(name: str, enc_name: str, *, result_writer=None) -> str: + enc = next(enc for enc in spec.inst_encodings if enc.enc_name == enc_name) + inst = next(inst for inst in enc.insts if inst.name == name) + generator._current_inst_fields = {field.name for field in enc.ucode_fields} + generator._current_operand_names = {operand.name for operand in inst.operands} + generator._current_enc = enc + return generator._gen_execute_body( + inst, + semantics.instructions[name], + enc.enc_name, + result_writer=result_writer, + ) + + vop3 = generated_body('V_ADD_NC_U32', 'ENC_VOP3') + assert 'vop3_integer_add' in vop3 + assert 'inst_.clamp' in vop3 + + subrev = generated_body('V_SUBREV_NC_U32', 'ENC_VOP3') + assert 'vop3_integer_sub' in subrev + assert subrev.index('read_lane(src1, lane)') < subrev.index('read_lane(src0, lane)') + assert 'inst_.clamp' in subrev + + vop2 = generated_body('V_ADD_NC_U32', 'ENC_VOP2') + assert 'vop3_integer_add' not in vop2 + assert 'inst_.clamp' not in vop2 + + carry = generated_body( + 'V_ADD_CO_U32', 'VOP3_SDST_ENC', result_writer='commit_result' + ) + assert 'inst_.clamp && w > 0xFFFFFFFFULL ? UINT32_MAX' in carry + assert 'inst_.clamp' in carry + assert 'if (w > 0xFFFFFFFFULL) vcc |=' in carry + + def test_noop_format_validation_is_inherited(amdgpu_generated_root: Path): encodings_h = (amdgpu_generated_root / 'rdna4' / 'encodings.h').read_text() diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py index 5eab503368fc..eb7d4240665d 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py @@ -1878,6 +1878,33 @@ def test_sub_co(self): } assert 'sub_co' in call_names + @pytest.mark.parametrize( + ('operation', 'expected'), + [ + ('add', 'inst_.clamp && w > 0xFFFFFFFFULL ? UINT32_MAX'), + ('sub', 'inst_.clamp && a < b ? 0u'), + ('rsub', 'inst_.clamp && a < b ? 0u'), + ('addc', 'inst_.clamp && w > 0xFFFFFFFFULL ? UINT32_MAX'), + ('subbc', 'inst_.clamp && a < b ? 0u'), + ('subbrevco', 'inst_.clamp && a < b ? 0u'), + ], + ) + def test_vop3_clamp_saturates_result_without_changing_carry( + self, operation, expected + ): + sem = _FakeSem('V_CARRY_U32', 'vector_add_co', operation, 'u32') + block = derive_sema_block(sem) + cpp = lower_sema_block( + block, + LoweringContext( + exec_model=block.pragma, + integer_saturation_dtype='u32', + ), + ) + + assert expected in cpp + assert 'vcc |= (1ULL << lane)' in cpp + class TestDeriveVectorCndmask: def test_cndmask(self): @@ -2451,6 +2478,29 @@ def test_pk_binop_bf16_generator_uses_bf16_helpers(self): assert 'util::f32_to_bf16' in cpp assert 'util::f32_to_f16' not in cpp + @pytest.mark.parametrize( + ('operation', 'data_type', 'helper'), + [ + ('add', 'i16', 'vop3_integer_add'), + ('sub', 'i16', 'vop3_integer_sub'), + ('add', 'u16', 'vop3_integer_add'), + ('sub', 'u16', 'vop3_integer_sub'), + ], + ) + def test_pk_integer_add_sub_clamps_each_selected_half( + self, operation, data_type, helper + ): + cpp = gen_pk_binop( + ['vdst'], + ['src0', 'src1'], + operation, + data_type, + ('inst_.opsel', 'inst_.opsel_hi'), + ) + + assert cpp.count(helper) == 2 + assert cpp.count('inst_.clamp') == 2 + def test_pk_ternary_bf16_generator_uses_fma_and_bf16_helpers(self): cpp = gen_pk_ternary( ['vdst'], diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp index 189e3ba345c4..3bca13cc720b 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp @@ -203,8 +203,10 @@ void VPkAddI16Vop3p::execute_impl(amdgpu::Wavefront &wf) { int16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); int16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); int16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo + b_lo); - uint16_t rhi = static_cast(a_hi + b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_add(a_lo, b_lo, inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_add(a_hi, b_hi, inst_.clamp)); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -225,8 +227,10 @@ void VPkSubI16Vop3p::execute_impl(amdgpu::Wavefront &wf) { int16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); int16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); int16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo - b_lo); - uint16_t rhi = static_cast(a_hi - b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_sub(a_lo, b_lo, inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_sub(a_hi, b_hi, inst_.clamp)); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -384,8 +388,10 @@ void VPkAddU16Vop3p::execute_impl(amdgpu::Wavefront &wf) { uint16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); uint16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo + b_lo); - uint16_t rhi = static_cast(a_hi + b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_add(a_lo, b_lo, inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_add(a_hi, b_hi, inst_.clamp)); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -406,8 +412,10 @@ void VPkSubU16Vop3p::execute_impl(amdgpu::Wavefront &wf) { uint16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); uint16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo - b_lo); - uint16_t rhi = static_cast(a_hi - b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_sub(a_lo, b_lo, inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_sub(a_hi, b_hi, inst_.clamp)); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3p_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3p_exec.cpp index 8ebb2ccfcd42..470468c7177d 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3p_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3p_exec.cpp @@ -88,8 +88,10 @@ void VPkAddI16Vop3p::execute_impl(amdgpu::Wavefront &wf) { int16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); int16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); int16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo + b_lo); - uint16_t rhi = static_cast(a_hi + b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_add(a_lo, b_lo, inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_add(a_hi, b_hi, inst_.clamp)); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -110,8 +112,10 @@ void VPkSubI16Vop3p::execute_impl(amdgpu::Wavefront &wf) { int16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); int16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); int16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo - b_lo); - uint16_t rhi = static_cast(a_hi - b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_sub(a_lo, b_lo, inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_sub(a_hi, b_hi, inst_.clamp)); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -269,8 +273,10 @@ void VPkAddU16Vop3p::execute_impl(amdgpu::Wavefront &wf) { uint16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); uint16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo + b_lo); - uint16_t rhi = static_cast(a_hi + b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_add(a_lo, b_lo, inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_add(a_hi, b_hi, inst_.clamp)); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -291,8 +297,10 @@ void VPkSubU16Vop3p::execute_impl(amdgpu::Wavefront &wf) { uint16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); uint16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo - b_lo); - uint16_t rhi = static_cast(a_hi - b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_sub(a_lo, b_lo, inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_sub(a_hi, b_hi, inst_.clamp)); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h index 154454df9190..25925ba8717d 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h @@ -2946,7 +2946,7 @@ inline void execute_v_add_co_ci_u32_vop3([[maybe_unused]] Inst &inst, vcc |= (1ULL << lane); else vcc &= ~(1ULL << lane); - return static_cast(w); + return inst.inst_.clamp && w > 0xFFFFFFFFULL ? UINT32_MAX : static_cast(w); }()); } commit_result(vcc); @@ -2996,7 +2996,7 @@ inline void execute_v_add_co_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse vcc |= (1ULL << lane); else vcc &= ~(1ULL << lane); - return static_cast(w); + return inst.inst_.clamp && w > 0xFFFFFFFFULL ? UINT32_MAX : static_cast(w); }()); } commit_result(vcc); @@ -3411,7 +3411,7 @@ inline void execute_v_addc_co_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unus vcc |= (1ULL << lane); else vcc &= ~(1ULL << lane); - return static_cast(w); + return inst.inst_.clamp && w > 0xFFFFFFFFULL ? UINT32_MAX : static_cast(w); }()); } commit_result(vcc); @@ -17175,8 +17175,10 @@ inline void execute_v_pk_add_i16_vop3p([[maybe_unused]] Inst &inst, int16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); int16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); int16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo + b_lo); - uint16_t rhi = static_cast(a_hi + b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_add(a_lo, b_lo, inst.inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_add(a_hi, b_hi, inst.inst_.clamp)); sdwa::write_lane(inst, wf, inst.vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -17204,8 +17206,10 @@ inline void execute_v_pk_add_u16_vop3p([[maybe_unused]] Inst &inst, uint16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); uint16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo + b_lo); - uint16_t rhi = static_cast(a_hi + b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_add(a_lo, b_lo, inst.inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_add(a_hi, b_hi, inst.inst_.clamp)); sdwa::write_lane(inst, wf, inst.vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -17893,8 +17897,10 @@ inline void execute_v_pk_sub_i16_vop3p([[maybe_unused]] Inst &inst, int16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); int16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); int16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo - b_lo); - uint16_t rhi = static_cast(a_hi - b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_sub(a_lo, b_lo, inst.inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_sub(a_hi, b_hi, inst.inst_.clamp)); sdwa::write_lane(inst, wf, inst.vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -17922,8 +17928,10 @@ inline void execute_v_pk_sub_u16_vop3p([[maybe_unused]] Inst &inst, uint16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); uint16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo - b_lo); - uint16_t rhi = static_cast(a_hi - b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_sub(a_lo, b_lo, inst.inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_sub(a_hi, b_hi, inst.inst_.clamp)); sdwa::write_lane(inst, wf, inst.vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -19020,14 +19028,14 @@ inline void execute_v_sub_co_ci_u32_vop3([[maybe_unused]] Inst &inst, continue; sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { uint64_t a = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), - b = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), - c = static_cast( - ((amdgpu::read_wave_mask_scalar(inst.src2, wf) >> lane) & 1)); - if (a < b + c) + b = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)) + + static_cast( + ((amdgpu::read_wave_mask_scalar(inst.src2, wf) >> lane) & 1)); + if (a < b) vcc |= (1ULL << lane); else vcc &= ~(1ULL << lane); - return static_cast(a - b - c); + return inst.inst_.clamp && a < b ? 0u : static_cast(a - b); }()); } commit_result(vcc); @@ -19067,13 +19075,13 @@ inline void execute_v_sub_co_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse if (!(exec & (1ULL << lane))) continue; sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { - uint32_t a = amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), - b = amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane); + uint64_t a = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + b = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); if (a < b) vcc |= (1ULL << lane); else vcc &= ~(1ULL << lane); - return a - b; + return inst.inst_.clamp && a < b ? 0u : static_cast(a - b); }()); } commit_result(vcc); @@ -19403,14 +19411,14 @@ inline void execute_v_subb_co_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unus continue; sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { uint64_t a = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), - b = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), - c = static_cast( - ((amdgpu::read_wave_mask_scalar(inst.src2, wf) >> lane) & 1)); - if (a < b + c) + b = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)) + + static_cast( + ((amdgpu::read_wave_mask_scalar(inst.src2, wf) >> lane) & 1)); + if (a < b) vcc |= (1ULL << lane); else vcc &= ~(1ULL << lane); - return static_cast(a - b - c); + return inst.inst_.clamp && a < b ? 0u : static_cast(a - b); }()); } commit_result(vcc); @@ -19464,14 +19472,14 @@ inline void execute_v_subbrev_co_u32_vop3([[maybe_unused]] Inst &inst, continue; sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { uint64_t a = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), - b = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), - c = static_cast( - ((amdgpu::read_wave_mask_scalar(inst.src2, wf) >> lane) & 1)); - if (a < b + c) + b = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)) + + static_cast( + ((amdgpu::read_wave_mask_scalar(inst.src2, wf) >> lane) & 1)); + if (a < b) vcc |= (1ULL << lane); else vcc &= ~(1ULL << lane); - return static_cast(a - b - c); + return inst.inst_.clamp && a < b ? 0u : static_cast(a - b); }()); } commit_result(vcc); @@ -19525,14 +19533,14 @@ inline void execute_v_subrev_co_ci_u32_vop3([[maybe_unused]] Inst &inst, continue; sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { uint64_t a = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), - b = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), - c = static_cast( - ((amdgpu::read_wave_mask_scalar(inst.src2, wf) >> lane) & 1)); - if (a < b + c) + b = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)) + + static_cast( + ((amdgpu::read_wave_mask_scalar(inst.src2, wf) >> lane) & 1)); + if (a < b) vcc |= (1ULL << lane); else vcc &= ~(1ULL << lane); - return static_cast(a - b - c); + return inst.inst_.clamp && a < b ? 0u : static_cast(a - b); }()); } commit_result(vcc); @@ -19574,13 +19582,13 @@ inline void execute_v_subrev_co_u32_vop3([[maybe_unused]] Inst &inst, if (!(exec & (1ULL << lane))) continue; sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { - uint32_t a = amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), - b = amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane); + uint64_t a = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + b = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); if (a < b) vcc |= (1ULL << lane); else vcc &= ~(1ULL << lane); - return a - b; + return inst.inst_.clamp && a < b ? 0u : static_cast(a - b); }()); } commit_result(vcc); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h index bcfd276a730d..c98ccd5a4e75 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h @@ -3376,8 +3376,8 @@ template [[nodiscard]] inline bool try_execute_binary_vop3_co_result_simd(Inst &inst, Wavefront &wf, CarryOp carry_op, WriteResult write_result) { - if (simd_force_scalar() || !inst.src0.simd_capable() || !inst.src1.simd_capable() || - !inst.vdst.simd_capable()) + if (simd_force_scalar() || inst.inst_.clamp || !inst.src0.simd_capable() || + !inst.src1.simd_capable() || !inst.vdst.simd_capable()) return false; using T = uint32_t; constexpr std::size_t W = util::native_width_v; @@ -3429,8 +3429,8 @@ template [[nodiscard]] inline bool try_execute_binary_vop3_cin_result_simd(Inst &inst, Wavefront &wf, CarryOp carry_op, WriteResult write_result) { - if (simd_force_scalar() || !inst.src0.simd_capable() || !inst.src1.simd_capable() || - !inst.vdst.simd_capable()) + if (simd_force_scalar() || inst.inst_.clamp || !inst.src0.simd_capable() || + !inst.src1.simd_capable() || !inst.vdst.simd_capable()) return false; using T = uint32_t; constexpr std::size_t W = util::native_width_v; @@ -3613,8 +3613,8 @@ template /// output) and op_sel_hi = 3 (both srcs feed their high half into the /// high output) — the LLVM-AS encoder emits this for the default-mode /// pk mnemonics, and the SIMD fast path bails when any other combination -/// is requested. The scalar bodies do NOT apply neg / neg_hi / clamp on -/// these integer pk ops, so the SIMD path also passes through. Functor +/// or integer saturation is requested. The scalar bodies apply CLAMP to +/// packed integer add/sub independently for each selected half. Functor /// receives the two source u32 lane vectors (each holding {low16, high16} /// packed) and returns the same shape; the per-half decompose / recompose /// lives inside the functor for op-specific flexibility (e.g. mul_lo @@ -3625,7 +3625,7 @@ template if (simd_force_scalar() || !sdwa::supports_direct_simd_store(inst) || !inst.src0.simd_capable() || !inst.src1.simd_capable() || !inst.vdst.simd_capable()) return false; - if (inst.inst_.op_sel != 0u || inst.inst_.op_sel_hi != 3u) + if (inst.inst_.clamp || inst.inst_.op_sel != 0u || inst.inst_.op_sel_hi != 3u) return false; using T = uint32_t; constexpr std::size_t W = util::native_width_v; diff --git a/emulation/rocjitsu/tests/cdna5_execution_test.cpp b/emulation/rocjitsu/tests/cdna5_execution_test.cpp index 8b7bf7f91db4..c3f12b457dfb 100644 --- a/emulation/rocjitsu/tests/cdna5_execution_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_execution_test.cpp @@ -185,13 +185,19 @@ TEST(Gfx1250ExecutionTest, Vop3IntegerClampSaturatesBeforeNarrowing) { uint32_t rhs; uint32_t expected; }; - constexpr std::array kBoundaryCases = {{ + constexpr std::array kBoundaryCases = {{ {cdna5::kVAddNcU32Vop3, UINT32_MAX, 1u, UINT32_MAX}, {cdna5::kVSubNcU32Vop3, 7u, 3u, 4u}, + {cdna5::kVSubrevNcU32Vop3, 3u, 7u, 4u}, + {cdna5::kVSubrevNcU32Vop3, 7u, 3u, 0u}, {cdna5::kVAddNcI32Vop3, static_cast(INT32_MAX), 1u, static_cast(INT32_MAX)}, + {cdna5::kVAddNcI32Vop3, static_cast(INT32_MIN), static_cast(-1), + static_cast(INT32_MIN)}, {cdna5::kVSubNcI32Vop3, static_cast(INT32_MIN), 1u, static_cast(INT32_MIN)}, + {cdna5::kVSubNcI32Vop3, static_cast(INT32_MAX), static_cast(-1), + static_cast(INT32_MAX)}, {cdna5::kVAddNcU16Vop3, UINT16_MAX, 1u, UINT16_MAX}, {cdna5::kVSubNcU16Vop3, 0u, 1u, 0u}, {cdna5::kVAddNcI16Vop3, static_cast(INT16_MAX), 1u, @@ -222,6 +228,90 @@ TEST(Gfx1250ExecutionTest, Vop3IntegerClampSaturatesBeforeNarrowing) { } } +TEST(Gfx1250ExecutionTest, Vop3PackedIntegerClampSaturatesSelectedHalves) { + ForceScalarGuard guard; + struct TestCase { + uint16_t opcode; + uint32_t src0; + uint32_t src1; + uint32_t expected; + }; + constexpr std::array kCases{ + TestCase{cdna5::kVPkAddI16Vop3p, 0x7fff8000u, 0x0001ffffu, 0x80007fffu}, + TestCase{cdna5::kVPkSubI16Vop3p, 0x7fff8000u, 0xffff0001u, 0x80007fffu}, + TestCase{cdna5::kVPkAddU16Vop3p, 0xffff0000u, 0x00010001u, 0x0001ffffu}, + TestCase{cdna5::kVPkSubU16Vop3p, 0x0000ffffu, 0x00010001u, 0xfffe0000u}, + }; + for (uint32_t scalar = 0; scalar < 2; ++scalar) { + util::set_force_scalar_for_testing(scalar != 0); + for (const TestCase &test : kCases) { + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + const uint32_t base = wf->vgpr_alloc().base; + cu->write_vgpr(base, 0, test.src0); + cu->write_vgpr(base + 1, 0, test.src1); + const auto words = cdna5::build_vop3p( + test.opcode, + {.vdst = 2, .opsel = 3, .clamp = 1, .src0 = 256, .src1 = 257, .opsel_hi = 0}); + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA5); + ASSERT_NE(decoder, nullptr); + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + inst->execute(*inst, wf); + EXPECT_EQ(cu->read_vgpr(base + 2, 0), test.expected) << "scalar " << scalar; + } + } +} + +TEST(Gfx1250ExecutionTest, Vop3CarryClampSaturatesVdstAndPreservesCarryBorrow) { + ForceScalarGuard guard; + struct TestCase { + uint16_t opcode; + uint32_t src0; + uint32_t src1; + bool carry_in; + }; + constexpr std::array kCases{ + TestCase{cdna5::kVAddCoU32Vop3SdstEnc, UINT32_MAX, 1u, false}, + TestCase{cdna5::kVSubCoU32Vop3SdstEnc, 0u, 1u, false}, + TestCase{cdna5::kVSubrevCoU32Vop3SdstEnc, 1u, 0u, false}, + TestCase{cdna5::kVAddCoCiU32Vop3SdstEnc, UINT32_MAX, 0u, true}, + TestCase{cdna5::kVSubCoCiU32Vop3SdstEnc, 0u, 0u, true}, + TestCase{cdna5::kVSubrevCoCiU32Vop3SdstEnc, 0u, 0u, true}, + }; + for (uint32_t scalar = 0; scalar < 2; ++scalar) { + util::set_force_scalar_for_testing(scalar != 0); + for (const TestCase &test : kCases) { + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + const uint32_t base = wf->vgpr_alloc().base; + cu->write_vgpr(base, 0, test.src0); + cu->write_vgpr(base + 1, 0, test.src1); + write_wave_sgpr(*cu, *wf, 4, test.carry_in ? 1u : 0u); + write_wave_sgpr(*cu, *wf, 2, 0u); + const auto words = cdna5::build_vop3_sdst_enc( + test.opcode, {.vdst = 2, .sdst = 2, .clamp = 1, .src0 = 256, .src1 = 257, .src2 = 4}); + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA5); + ASSERT_NE(decoder, nullptr); + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + inst->execute(*inst, wf); + EXPECT_EQ(cu->read_vgpr(base + 2, 0), test.opcode == cdna5::kVAddCoU32Vop3SdstEnc || + test.opcode == cdna5::kVAddCoCiU32Vop3SdstEnc + ? UINT32_MAX + : 0u) + << "scalar " << scalar; + EXPECT_EQ(read_wave_sgpr(*cu, *wf, 2) & 1u, 1u) << "scalar " << scalar; + } + } +} + TEST(FpModePolicyTest, F64HelpersRestoreAmbientHostEnvironment) { HostFenvGuard environment_guard; ASSERT_EQ(std::fesetround(FE_DOWNWARD), 0); From 4be5d7bf9b52f90d3a6fbdbe1623938423a9a565 Mon Sep 17 00:00:00 2001 From: Sambhav Jain Date: Wed, 2 Sep 2026 17:42:23 +0000 Subject: [PATCH 03/16] [rocjitsu] Saturate clamped u64 VOP3 arithmetic Extend the destination-width-aware CLAMP helpers to unsigned 64-bit add and subtract so gfx1250 normal and DPP execution cannot wrap. Regenerate the execute paths and cover both saturation boundaries. Signed-off-by: Sambhav Jain --- .../lib/python/amdisa/codegen/_generator.py | 2 +- .../amdisa/codegen/execute/sema_lower.py | 2 +- .../tests/test_generator_profile_gates.py | 8 +++ .../python/amdisa/tests/test_sema_derive.py | 2 + .../amdgpu/generated/cdna5/vop3_exec_alu.cpp | 24 ++++--- .../isa/arch/amdgpu/shared/simd_glue.h | 37 ++++++----- .../rocjitsu/tests/cdna5_execution_test.cpp | 62 +++++++++++++++++++ 7 files changed, 111 insertions(+), 26 deletions(-) diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py index bc39f5ef421f..edcf81751276 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py @@ -6156,7 +6156,7 @@ def _semantic_reg_class(opnd: Operand) -> RegClass: ( cls == 'vector_binop' and op in ('add', 'sub', 'subrev', 'rsub') - and dtype in ('i16', 'u16', 'i32', 'u32') + and dtype in ('i16', 'u16', 'i32', 'u32', 'u64') ) or (cls == 'vector_add_co' and dtype == 'u32') ) diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py index 9e8039890764..8d20a142b971 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py @@ -737,7 +737,7 @@ def _lower_expr(node: SemaNode, ctx: LoweringContext) -> str: def _lower_saturating_integer_binary(node: SemaNode, ctx: LoweringContext) -> str: """Lower VOP3 integer add/sub with optional CLAMP saturation.""" dtype = ctx.integer_saturation_dtype - if dtype not in ('i16', 'u16', 'i32', 'u32'): + if dtype not in ('i16', 'u16', 'i32', 'u32', 'u64'): raise ValueError(f'Unsupported integer saturation type: {dtype}') lhs = _lower_expr(node.children[0], replace(ctx, integer_saturation_dtype=None)) diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py index 4184c70cb1ee..ab1049c19a68 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py @@ -4106,6 +4106,14 @@ def generated_body(name: str, enc_name: str, *, result_writer=None) -> str: assert subrev.index('read_lane(src1, lane)') < subrev.index('read_lane(src0, lane)') assert 'inst_.clamp' in subrev + add_u64 = generated_body('V_ADD_NC_U64', 'ENC_VOP3') + assert 'vop3_integer_add' in add_u64 + assert 'inst_.clamp' in add_u64 + + sub_u64 = generated_body('V_SUB_NC_U64', 'ENC_VOP3') + assert 'vop3_integer_sub' in sub_u64 + assert 'inst_.clamp' in sub_u64 + vop2 = generated_body('V_ADD_NC_U32', 'ENC_VOP2') assert 'vop3_integer_add' not in vop2 assert 'inst_.clamp' not in vop2 diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py index eb7d4240665d..0084be90252e 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py @@ -1589,6 +1589,8 @@ def test_signed_add_sub_use_unsigned_operands_to_avoid_ub(self): ('sub', 'i32', 'vop3_integer_sub'), ('add', 'u16', 'vop3_integer_add'), ('sub', 'i16', 'vop3_integer_sub'), + ('add', 'u64', 'vop3_integer_add'), + ('sub', 'u64', 'vop3_integer_sub'), ], ) def test_vop3_integer_clamp_saturates_before_narrowing(self, op, dtype, expected): diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_alu.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_alu.cpp index b5786f33c56c..543c7867c905 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_alu.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_alu.cpp @@ -4033,8 +4033,10 @@ void VAddNcU64Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane64( *this, wf, vdst, lane, - (static_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)) + - static_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)))); + amdgpu::vop3_integer_add( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)), + inst_.clamp)); } } @@ -4058,8 +4060,10 @@ RJ_NOINLINE void VAddNcU64Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane64( *this, wf, vdst, lane, - (static_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)) + - static_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)))); + amdgpu::vop3_integer_add( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)), + inst_.clamp)); } dpp_write_mask_scope_.restore(); } @@ -4075,8 +4079,10 @@ void VSubNcU64Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane64( *this, wf, vdst, lane, - (static_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)) - - static_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)))); + amdgpu::vop3_integer_sub( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)), + inst_.clamp)); } } @@ -4100,8 +4106,10 @@ RJ_NOINLINE void VSubNcU64Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane64( *this, wf, vdst, lane, - (static_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)) - - static_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)))); + amdgpu::vop3_integer_sub( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)), + inst_.clamp)); } dpp_write_mask_scope_.restore(); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h index c98ccd5a4e75..ef230d5409d1 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h @@ -96,44 +96,49 @@ template inline T clamp_floating_result(T value, const Wavefront &w } /// @brief Execute VOP3 integer addition, saturating when CLAMP is set. -template inline uint32_t vop3_integer_add(uint32_t lhs, uint32_t rhs, bool clamp) { - static_assert(std::is_integral_v && (sizeof(T) == 2 || sizeof(T) == 4)); +template +inline std::make_unsigned_t vop3_integer_add(std::make_unsigned_t lhs, + std::make_unsigned_t rhs, bool clamp) { + static_assert(std::is_integral_v && (sizeof(T) == 2 || sizeof(T) == 4 || sizeof(T) == 8)); using U = std::make_unsigned_t; - const U lhs_bits = static_cast(lhs); - const U rhs_bits = static_cast(rhs); + const U lhs_bits = lhs; + const U rhs_bits = rhs; if (!clamp) - return static_cast(static_cast(lhs_bits + rhs_bits)); + return static_cast(lhs_bits + rhs_bits); if constexpr (std::is_signed_v) { + static_assert(sizeof(T) <= 4, "signed 64-bit VOP3 saturation is not implemented"); using Wide = std::conditional_t; const Wide wide = static_cast(std::bit_cast(lhs_bits)) + static_cast(std::bit_cast(rhs_bits)); const Wide saturated = std::clamp(wide, static_cast(std::numeric_limits::min()), static_cast(std::numeric_limits::max())); - return static_cast(static_cast(saturated)); + return static_cast(saturated); } else { - using Wide = std::conditional_t; - const Wide wide = static_cast(lhs_bits) + static_cast(rhs_bits); - return static_cast(std::min(wide, static_cast(std::numeric_limits::max()))); + const U max = std::numeric_limits::max(); + return rhs_bits > max - lhs_bits ? max : static_cast(lhs_bits + rhs_bits); } } /// @brief Execute VOP3 integer subtraction, saturating when CLAMP is set. -template inline uint32_t vop3_integer_sub(uint32_t lhs, uint32_t rhs, bool clamp) { - static_assert(std::is_integral_v && (sizeof(T) == 2 || sizeof(T) == 4)); +template +inline std::make_unsigned_t vop3_integer_sub(std::make_unsigned_t lhs, + std::make_unsigned_t rhs, bool clamp) { + static_assert(std::is_integral_v && (sizeof(T) == 2 || sizeof(T) == 4 || sizeof(T) == 8)); using U = std::make_unsigned_t; - const U lhs_bits = static_cast(lhs); - const U rhs_bits = static_cast(rhs); + const U lhs_bits = lhs; + const U rhs_bits = rhs; if (!clamp) - return static_cast(static_cast(lhs_bits - rhs_bits)); + return static_cast(lhs_bits - rhs_bits); if constexpr (std::is_signed_v) { + static_assert(sizeof(T) <= 4, "signed 64-bit VOP3 saturation is not implemented"); using Wide = std::conditional_t; const Wide wide = static_cast(std::bit_cast(lhs_bits)) - static_cast(std::bit_cast(rhs_bits)); const Wide saturated = std::clamp(wide, static_cast(std::numeric_limits::min()), static_cast(std::numeric_limits::max())); - return static_cast(static_cast(saturated)); + return static_cast(saturated); } else { - return lhs_bits < rhs_bits ? 0u : static_cast(lhs_bits - rhs_bits); + return lhs_bits < rhs_bits ? U{0} : static_cast(lhs_bits - rhs_bits); } } diff --git a/emulation/rocjitsu/tests/cdna5_execution_test.cpp b/emulation/rocjitsu/tests/cdna5_execution_test.cpp index c3f12b457dfb..d1c9b298f67d 100644 --- a/emulation/rocjitsu/tests/cdna5_execution_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_execution_test.cpp @@ -228,6 +228,68 @@ TEST(Gfx1250ExecutionTest, Vop3IntegerClampSaturatesBeforeNarrowing) { } } +TEST(Gfx1250ExecutionTest, Vop3U64ClampSaturatesNormalAndDppPaths) { + constexpr uint32_t kSrc0 = 0; + constexpr uint32_t kSrc1 = 2; + constexpr uint32_t kDst = 4; + + auto run_case = [](uint16_t opcode, uint64_t lhs, uint64_t rhs, uint64_t expected, bool dpp) { + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + const uint32_t base = wf->vgpr_alloc().base; + auto write_vgpr64 = [&](uint32_t reg, uint64_t value) { + cu->write_vgpr(base + reg, 0, static_cast(value)); + cu->write_vgpr(base + reg + 1, 0, static_cast(value >> 32)); + }; + auto read_vgpr64 = [&](uint32_t reg) { + return static_cast(cu->read_vgpr(base + reg, 0)) | + (static_cast(cu->read_vgpr(base + reg + 1, 0)) << 32); + }; + write_vgpr64(kSrc0, lhs); + write_vgpr64(kSrc1, rhs); + write_vgpr64(kDst, 0u); + + if (!dpp) { + const auto words = cdna5::build_vop3( + opcode, {.vdst = kDst, .clamp = 1, .src0 = 256 + kSrc0, .src1 = 256 + kSrc1}); + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA5); + ASSERT_NE(decoder, nullptr); + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + inst->execute(*inst, wf); + } else { + cdna5::Vop3VopDpp16MachineInst raw{}; + raw.vdst = kDst; + raw.clamp = 1; + raw.src0 = amdgpu::SRC_DPP; + raw.src1 = 256 + kSrc1; + raw.vsrc0 = kSrc0; + raw.dpp_ctrl = amdgpu::dpp::ROW_SELECT_BASE; + raw.bound_ctrl = 1; + raw.bank_mask = 0xF; + raw.row_mask = 0xF; + if (opcode == cdna5::kVAddNcU64Vop3) { + cdna5::VAddNcU64Vop3 inst(reinterpret_cast(&raw)); + inst.execute_impl(*wf); + } else { + ASSERT_EQ(opcode, cdna5::kVSubNcU64Vop3); + cdna5::VSubNcU64Vop3 inst(reinterpret_cast(&raw)); + inst.execute_impl(*wf); + } + } + + EXPECT_EQ(read_vgpr64(kDst), expected) << "dpp " << dpp; + }; + + for (bool dpp : {false, true}) { + run_case(cdna5::kVAddNcU64Vop3, UINT64_MAX, 1u, UINT64_MAX, dpp); + run_case(cdna5::kVSubNcU64Vop3, 0u, 1u, 0u, dpp); + } +} + TEST(Gfx1250ExecutionTest, Vop3PackedIntegerClampSaturatesSelectedHalves) { ForceScalarGuard guard; struct TestCase { From 26e13e710ac68c9ede1180cc17f206b327df14ef Mon Sep 17 00:00:00 2001 From: Sambhav Jain Date: Wed, 2 Sep 2026 18:22:56 +0000 Subject: [PATCH 04/16] [rocjitsu] Saturate clamped ternary integer addition Route clamp-capable V_ADD3_U32 through full-width unsigned saturation and bypass the wrapping SIMD path when CLAMP is set. Add generator coverage and RDNA4 normal/DPP execution regressions for the overflow boundary. Signed-off-by: Sambhav Jain --- .../lib/python/amdisa/codegen/_generator.py | 1 + .../amdisa/codegen/execute/sema_lower.py | 3 + .../tests/test_generator_profile_gates.py | 26 +++++++ .../python/amdisa/tests/test_sema_derive.py | 14 ++++ .../amdgpu/generated/shared/execute_shared.h | 10 +-- .../isa/arch/amdgpu/shared/simd_glue.h | 24 +++++-- ...vop3_ternary_int_simd_correctness_test.cpp | 70 ++++++++++++++++++- 7 files changed, 134 insertions(+), 14 deletions(-) diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py index edcf81751276..0eaccc4744e1 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py @@ -6159,6 +6159,7 @@ def _semantic_reg_class(opnd: Operand) -> RegClass: and dtype in ('i16', 'u16', 'i32', 'u32', 'u64') ) or (cls == 'vector_add_co' and dtype == 'u32') + or (cls == 'vector_ternary' and op == 'add3' and dtype == 'u32') ) ): lctx.integer_saturation_dtype = dtype diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py index 8d20a142b971..387883fbfb7b 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py @@ -1751,6 +1751,9 @@ def _lower_call(node: SemaNode, ctx: LoweringContext) -> str: args = [_lower_expr(c, ctx) for c in node.children[1:]] args_str = ', '.join(args) + if ctx.integer_saturation_dtype == 'u32' and callee == 'add3': + return f'amdgpu::vop3_integer_add3({args_str}, inst_.clamp)' + if ctx.integer_saturation_dtype == 'u32' and callee in ( 'add_co', 'sub_co', diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py index ab1049c19a68..cdac71dfa85e 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py @@ -4126,6 +4126,32 @@ def generated_body(name: str, enc_name: str, *, result_writer=None) -> str: assert 'if (w > 0xFFFFFFFFULL) vcc |=' in carry +def test_rdna4_generator_wires_add3_clamp_from_encoded_vop3_field(): + isa_xml = _mrisa_dir() / 'amdgpu_isa_rdna4.xml' + parser = Parser(str(isa_xml), Rdna4Profile()) + spec = parser.parse() + semantics = derive_all_semantics(spec) + generator = CodeGenerator(spec, '', semantics) + enc = next(enc for enc in spec.inst_encodings if enc.enc_name == 'ENC_VOP3') + generator._current_inst_fields = {field.name for field in enc.ucode_fields} + generator._current_enc = enc + + def generated_body(name: str) -> str: + inst = next(inst for inst in enc.insts if inst.name == name) + generator._current_operand_names = {operand.name for operand in inst.operands} + return generator._gen_execute_body( + inst, semantics.instructions[name], enc.enc_name + ) + + add3 = generated_body('V_ADD3_U32') + assert 'vop3_integer_add3' in add3 + assert 'inst_.clamp' in add3 + + or3 = generated_body('V_OR3_B32') + assert 'vop3_integer_add3' not in or3 + assert 'inst_.clamp' not in or3 + + def test_noop_format_validation_is_inherited(amdgpu_generated_root: Path): encodings_h = (amdgpu_generated_root / 'rdna4' / 'encodings.h').read_text() diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py index 0084be90252e..6ba971bbf37b 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py @@ -1606,6 +1606,20 @@ def test_vop3_integer_clamp_saturates_before_narrowing(self, op, dtype, expected assert 'inst_.clamp' in cpp assert expected in cpp + def test_vop3_integer_add3_clamps_the_full_unsigned_sum(self): + sem = _FakeSem('V_ADD3_U32', 'vector_ternary', 'add3', 'u32') + block = derive_sema_block(sem) + cpp = lower_sema_block( + block, + LoweringContext( + exec_model=block.pragma, + integer_saturation_dtype='u32', + ), + ) + + assert 'vop3_integer_add3' in cpp + assert 'inst_.clamp' in cpp + def test_lshlrev(self): sem = _FakeSem('V_LSHLREV_B32', 'vector_binop', 'lshlrev', 'b32') block = derive_sema_block(sem) diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h index 25925ba8717d..30ead02a0278 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h @@ -2884,10 +2884,12 @@ inline void execute_v_add3_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused] for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) + - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane) + - amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_add3(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane), + inst.inst_.clamp)); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h index ef230d5409d1..c25bb8711e73 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h @@ -119,6 +119,15 @@ inline std::make_unsigned_t vop3_integer_add(std::make_unsigned_t lhs, } } +/// @brief Execute unsigned VOP3 integer three-input addition, saturating when CLAMP is set. +template +inline std::make_unsigned_t vop3_integer_add3(std::make_unsigned_t lhs, + std::make_unsigned_t rhs, + std::make_unsigned_t addend, bool clamp) { + static_assert(std::is_integral_v && std::is_unsigned_v); + return vop3_integer_add(vop3_integer_add(lhs, rhs, clamp), addend, clamp); +} + /// @brief Execute VOP3 integer subtraction, saturating when CLAMP is set. template inline std::make_unsigned_t vop3_integer_sub(std::make_unsigned_t lhs, @@ -2279,16 +2288,17 @@ template /// VOP3 integer/bitwise ternary SIMD fast path. Reads `src0`/`src1`/`src2`, /// runs `tern_op(a, b, c)`, and masked-stores the result. The generated scalar -/// bodies for these ternary integer ops apply no source/result modifiers (abs/ -/// neg/omod are float-only; clamp is unused on integer 3-source ops), so the -/// plain functor is bit-identical to the scalar body. T is a 32-bit integer -/// lane type (typically uint32_t). Same SIMD-capable / EXEC-chunk loop as the -/// binary VOP3 path. +/// bodies for these ternary integer ops apply no source/result modifiers except +/// integer CLAMP. Saturating arithmetic falls back to the scalar body; the plain +/// functor is bit-identical to that body when CLAMP is clear. T is a 32-bit +/// integer lane type (typically uint32_t). Same SIMD-capable / EXEC-chunk loop +/// as the binary VOP3 path. template requires(util::has_stdx_simd) [[nodiscard]] inline bool try_execute_ternary_vop3_simd(Inst &inst, Wavefront &wf, TernOp tern_op) { - if (simd_force_scalar() || !sdwa::supports_direct_simd_store(inst) || !inst.src0.simd_capable() || - !inst.src1.simd_capable() || !inst.src2.simd_capable() || !inst.vdst.simd_capable()) + if (inst.inst_.clamp != 0u || simd_force_scalar() || !sdwa::supports_direct_simd_store(inst) || + !inst.src0.simd_capable() || !inst.src1.simd_capable() || !inst.src2.simd_capable() || + !inst.vdst.simd_capable()) return false; constexpr std::size_t W = util::native_width_v; const uint64_t chunk_full = util::mask(static_cast(W)); diff --git a/emulation/rocjitsu/tests/simd_correctness/vop3_ternary_int_simd_correctness_test.cpp b/emulation/rocjitsu/tests/simd_correctness/vop3_ternary_int_simd_correctness_test.cpp index 62652d917517..1f3ee2db47cc 100644 --- a/emulation/rocjitsu/tests/simd_correctness/vop3_ternary_int_simd_correctness_test.cpp +++ b/emulation/rocjitsu/tests/simd_correctness/vop3_ternary_int_simd_correctness_test.cpp @@ -16,6 +16,10 @@ #include "util/simd_test_hooks.h" #include "rocjitsu/code/rj_code.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna4/builders.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna4/execution_backend.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna4/opcodes.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3.h" #include "rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h" #include "rocjitsu/isa/decoder.h" #include "rocjitsu/isa/instruction.h" @@ -116,15 +120,16 @@ struct Fixture { std::unique_ptr decoder; amdgpu::Wavefront *wf = nullptr; - Fixture() : gpu_mem("vop3_tern_int_mem"), l2("vop3_tern_int_l2") { + explicit Fixture(rj_code_arch_t arch = ROCJITSU_CODE_ARCH_CDNA4) + : gpu_mem("vop3_tern_int_mem"), l2("vop3_tern_int_l2") { amdgpu::ComputeUnitCore::Config cfg{}; - cfg.arch = ROCJITSU_CODE_ARCH_CDNA4; + cfg.arch = arch; cfg.num_wf_slots = 1; cfg.sgprs_per_wf = SGPRS_PER_WF; cfg.vgprs_per_wf = VGPRS_PER_WF; cfg.lds_size_kb = 64; cu = amdgpu::ComputeUnitCore::create("cu_vop3_tern_int", cfg, &gpu_mem, &l2); - decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4); + decoder = Decoder::create(arch); wf = cu->dispatch_wf(0, 0, SGPRS_PER_WF, VGPRS_PER_WF); } @@ -214,6 +219,65 @@ TEST(Vop3TernaryIntSimdCorrectness, PartialExec) { check_case(c, /*exec=*/0xA5A5'F0F0'1234'8001ULL); } +TEST(Vop3TernaryIntSimdCorrectness, Rdna4Add3ClampSaturatesNormalAndDppPaths) { + ForceScalarGuard gate_guard; + ScopedIsaExecutionBackend execution_backend_scope{&rdna4::execution_backend()}; + constexpr uint32_t kSrc0 = 0; + constexpr uint32_t kSrc1 = 1; + constexpr uint32_t kSrc2 = 2; + + auto run_case = [&](bool force_scalar, bool dpp, bool clamp) { + util::set_force_scalar_for_testing(force_scalar); + Fixture fx(ROCJITSU_CODE_ARCH_RDNA4); + EXPECT_NE(fx.cu, nullptr); + EXPECT_NE(fx.wf, nullptr); + if (fx.cu == nullptr || fx.wf == nullptr) + return 0u; + const uint32_t base = fx.wf->vgpr_alloc().base; + fx.wf->set_exec(1u); + fx.cu->write_vgpr(base + kSrc0, 0, UINT32_MAX); + fx.cu->write_vgpr(base + kSrc1, 0, 1u); + fx.cu->write_vgpr(base + kSrc2, 0, 0u); + fx.cu->write_vgpr(base + kDstVgpr, 0, DST_SENTINEL); + + if (!dpp) { + const auto words = + rdna4::build_vop3(rdna4::kVAdd3U32Vop3, {.vdst = kDstVgpr, + .clamp = static_cast(clamp), + .src0 = 256 + kSrc0, + .src1 = 256 + kSrc1, + .src2 = 256 + kSrc2}); + std::unique_ptr inst(decode_valid(*fx.decoder, words.data())); + EXPECT_NE(inst, nullptr); + if (inst == nullptr) + return 0u; + fx.cu->execute_instruction(inst.get(), *fx.wf); + } else { + rdna4::Vop3VopDpp16MachineInst raw{}; + raw.vdst = kDstVgpr; + raw.clamp = clamp; + raw.src0 = amdgpu::SRC_DPP; + raw.src1 = 256 + kSrc1; + raw.src2 = 256 + kSrc2; + raw.vsrc0 = kSrc0; + raw.dpp_ctrl = amdgpu::dpp::ROW_SELECT_BASE; + raw.bound_ctrl = 1; + raw.bank_mask = 0xF; + raw.row_mask = 0xF; + rdna4::VAdd3U32Vop3 inst(reinterpret_cast(&raw)); + inst.execute_impl(*fx.wf); + } + + return fx.cu->read_vgpr(base + kDstVgpr, 0); + }; + + for (bool force_scalar : {false, true}) { + EXPECT_EQ(run_case(force_scalar, false, true), UINT32_MAX); + EXPECT_EQ(run_case(force_scalar, true, true), UINT32_MAX); + EXPECT_EQ(run_case(force_scalar, false, false), 0u); + } +} + TEST(Vop3TernaryIntSimdCorrectness, IsaGoldenByteOperations) { struct GoldenCase { const char *name; From 9107eebf329e9bda4c28324c32b1b2c9c5fb92c9 Mon Sep 17 00:00:00 2001 From: Sambhav Jain Date: Wed, 2 Sep 2026 19:03:24 +0000 Subject: [PATCH 05/16] [rocjitsu] Gate integer clamp by instruction Limit integer clamp lowering to assembler-legal opcodes and remove unsupported V_ADD3_U32 behavior. Use exact wide intermediates for multiply-accumulate operations and cover scalar and SIMD saturation boundaries. Signed-off-by: Sambhav Jain --- .../lib/python/amdisa/codegen/_generator.py | 27 +--- .../amdisa/codegen/execute/sema_lower.py | 39 +++++- .../amdisa/codegen/execute/vector_special.py | 17 ++- .../rocjitsu/lib/python/amdisa/isa_profile.py | 53 +++++++ .../tests/test_generator_profile_gates.py | 25 +++- .../python/amdisa/tests/test_sema_derive.py | 23 ++- .../arch/amdgpu/generated/cdna1/vop3_exec.cpp | 17 ++- .../arch/amdgpu/generated/cdna2/vop3_exec.cpp | 17 ++- .../arch/amdgpu/generated/cdna3/vop3_exec.cpp | 17 ++- .../arch/amdgpu/generated/cdna4/vop3_exec.cpp | 17 ++- .../generated/cdna5/vop3_exec_ternary.cpp | 34 +++-- .../arch/amdgpu/generated/rdna3/vop3_exec.cpp | 34 +++-- .../amdgpu/generated/rdna3_5/vop3_exec.cpp | 34 +++-- .../arch/amdgpu/generated/rdna4/vop3_exec.cpp | 34 +++-- .../amdgpu/generated/shared/execute_shared.h | 132 ++++++++---------- .../isa/arch/amdgpu/shared/simd_glue.h | 77 ++++++++-- .../rocjitsu/tests/cdna5_execution_test.cpp | 52 +++++++ ...vop3_ternary_int_simd_correctness_test.cpp | 59 -------- 18 files changed, 423 insertions(+), 285 deletions(-) diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py index 0eaccc4744e1..494c3ab091f5 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py @@ -6149,20 +6149,9 @@ def _semantic_reg_class(opnd: Operand) -> RegClass: mask_result_writer=result_writer, ) inst_fields = getattr(self, '_current_inst_fields', set()) - if ( - is_vop3 - and 'clamp' in inst_fields - and ( - ( - cls == 'vector_binop' - and op in ('add', 'sub', 'subrev', 'rsub') - and dtype in ('i16', 'u16', 'i32', 'u32', 'u64') - ) - or (cls == 'vector_add_co' and dtype == 'u32') - or (cls == 'vector_ternary' and op == 'add3' and dtype == 'u32') - ) - ): - lctx.integer_saturation_dtype = dtype + integer_clamp_dtype = profile.integer_clamp_dtypes.get(inst.name) + if is_vop3 and 'clamp' in inst_fields and integer_clamp_dtype: + lctx.integer_saturation_dtype = integer_clamp_dtype if cls == 'vector_cmp': # V_CMP writes a fresh wave mask initialized to zero, so false # lanes can remain clear without emitting redundant bit clears. @@ -6281,17 +6270,13 @@ def _semantic_reg_class(opnd: Operand) -> RegClass: ), 'V_MAD_I16': ( 3, - ( - 'int32_t a = static_cast(s0);', - 'int32_t b = static_cast(s1);', - 'int32_t c = static_cast(s2);', - ), - 'static_cast(static_cast(a * b + c))', + (), + 'amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)', ), 'V_MAD_U16': ( 3, (), - '(s0 * s1 + s2) & 0xffffu', + 'amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)', ), } if is_true16_vop3 and inst.name in true16_special_vop3_ops: diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py index 387883fbfb7b..76d972ea77e4 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py @@ -740,8 +740,20 @@ def _lower_saturating_integer_binary(node: SemaNode, ctx: LoweringContext) -> st if dtype not in ('i16', 'u16', 'i32', 'u32', 'u64'): raise ValueError(f'Unsupported integer saturation type: {dtype}') - lhs = _lower_expr(node.children[0], replace(ctx, integer_saturation_dtype=None)) - rhs = _lower_expr(node.children[1], replace(ctx, integer_saturation_dtype=None)) + child_ctx = replace(ctx, integer_saturation_dtype=None) + if node.kind == SemaNodeKind.ADD and node.children[0].kind == SemaNodeKind.MUL: + product = node.children[0] + lhs = _lower_expr(product.children[0], child_ctx) + rhs = _lower_expr(product.children[1], child_ctx) + addend = _lower_expr(node.children[1], child_ctx) + cpp_type = f'{"int" if dtype.startswith("i") else "uint"}{dtype[1:]}_t' + return ( + f'amdgpu::vop3_integer_mad<{cpp_type}, {dtype[1:]}>(' + f'{lhs}, {rhs}, {addend}, inst_.clamp)' + ) + + lhs = _lower_expr(node.children[0], child_ctx) + rhs = _lower_expr(node.children[1], child_ctx) operation = 'add' if node.kind == SemaNodeKind.ADD else 'sub' cpp_type = f'{"int" if dtype.startswith("i") else "uint"}{dtype[1:]}_t' return ( @@ -1751,8 +1763,27 @@ def _lower_call(node: SemaNode, ctx: LoweringContext) -> str: args = [_lower_expr(c, ctx) for c in node.children[1:]] args_str = ', '.join(args) - if ctx.integer_saturation_dtype == 'u32' and callee == 'add3': - return f'amdgpu::vop3_integer_add3({args_str}, inst_.clamp)' + saturating_calls = { + 'mul_i24': ('int32_t', 24, False), + 'mul_u24': ('uint32_t', 24, False), + 'mad_i24': ('int32_t', 24, True), + 'mad_u24': ('uint32_t', 24, True), + 'mad_lo_u16': ('uint16_t', 16, True), + } + if ctx.integer_saturation_dtype is not None and callee in saturating_calls: + cpp_type, source_bits, has_addend = saturating_calls[callee] + helper = 'vop3_integer_mad' if has_addend else 'vop3_integer_mul' + return ( + f'amdgpu::{helper}<{cpp_type}, {source_bits}>(' f'{args_str}, inst_.clamp)' + ) + + if ctx.integer_saturation_dtype == 'u32' and callee in ( + 'sad_u8', + 'sad_u16', + 'sad_u32', + 'msad_u8', + ): + return f'amdgpu::vop3_integer_{callee}({args_str}, inst_.clamp)' if ctx.integer_saturation_dtype == 'u32' and callee in ( 'add_co', diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py index b163d9f96ab6..8e14d8d4ddaf 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py @@ -803,8 +803,16 @@ def gen_vector_mad_32_16( L.append( f' int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane({src[2]}, lane));' ) + result = ( + 'amdgpu::vop3_integer_mad(' + 'static_cast(s0), static_cast(s1), ' + 'static_cast(s2), inst_.clamp)' + if is_vop3 + else 'static_cast(s0) * static_cast(s1) + ' + 'static_cast(s2)' + ) L.append( - f' amdgpu::RegisterAccess(wf).write_lane({dst[0]}, lane, static_cast(s0) * static_cast(s1) + static_cast(s2));' + f' amdgpu::RegisterAccess(wf).write_lane({dst[0]}, lane, {result});' ) else: if is_vop3: @@ -820,8 +828,13 @@ def gen_vector_mad_32_16( L.append( f' uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane({src[2]}, lane);' ) + result = ( + 'amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)' + if is_vop3 + else 's0 * s1 + s2' + ) L.append( - f' amdgpu::RegisterAccess(wf).write_lane({dst[0]}, lane, s0 * s1 + s2);' + f' amdgpu::RegisterAccess(wf).write_lane({dst[0]}, lane, {result});' ) L.append(' }') return '\n'.join(L) diff --git a/emulation/rocjitsu/lib/python/amdisa/isa_profile.py b/emulation/rocjitsu/lib/python/amdisa/isa_profile.py index bdc0e1502da9..dc3771e1325e 100644 --- a/emulation/rocjitsu/lib/python/amdisa/isa_profile.py +++ b/emulation/rocjitsu/lib/python/amdisa/isa_profile.py @@ -580,6 +580,59 @@ def vop3p_source_modifier_omissions(self) -> frozenset[str]: """VOP3P instructions that do not use packed source modifier fields.""" return frozenset() + @property + def integer_clamp_dtypes(self) -> dict[str, str]: + """Integer VOP3 instructions whose CLAMP modifier saturates the result. + + CLAMP is present in the shared VOP3 encoding, but it is not legal for + every opcode in that encoding. Keep the instruction-level policy here + instead of inferring support from the encoding field alone. + """ + return { + 'V_ADDC_CO_U32': 'u32', + 'V_ADD_CO_CI_U32': 'u32', + 'V_ADD_CO_U32': 'u32', + 'V_ADD_I16': 'i16', + 'V_ADD_I32': 'i32', + 'V_ADD_NC_I16': 'i16', + 'V_ADD_NC_I32': 'i32', + 'V_ADD_NC_U16': 'u16', + 'V_ADD_NC_U32': 'u32', + 'V_ADD_NC_U64': 'u64', + 'V_ADD_U16': 'u16', + 'V_ADD_U32': 'u32', + 'V_MAD_I16': 'i16', + 'V_MAD_I32_I16': 'i32', + 'V_MAD_I32_I24': 'i32', + 'V_MAD_U16': 'u16', + 'V_MAD_U32_U16': 'u32', + 'V_MAD_U32_U24': 'u32', + 'V_MSAD_U8': 'u32', + 'V_MUL_I32_I24': 'i32', + 'V_MUL_U32_U24': 'u32', + 'V_SAD_U8': 'u32', + 'V_SAD_U16': 'u32', + 'V_SAD_U32': 'u32', + 'V_SUBBREV_CO_U32': 'u32', + 'V_SUBB_CO_U32': 'u32', + 'V_SUBREV_CO_CI_U32': 'u32', + 'V_SUBREV_CO_U32': 'u32', + 'V_SUBREV_NC_U32': 'u32', + 'V_SUBREV_U16': 'u16', + 'V_SUBREV_U32': 'u32', + 'V_SUB_CO_CI_U32': 'u32', + 'V_SUB_CO_U32': 'u32', + 'V_SUB_I16': 'i16', + 'V_SUB_I32': 'i32', + 'V_SUB_NC_I16': 'i16', + 'V_SUB_NC_I32': 'i32', + 'V_SUB_NC_U16': 'u16', + 'V_SUB_NC_U32': 'u32', + 'V_SUB_NC_U64': 'u64', + 'V_SUB_U16': 'u16', + 'V_SUB_U32': 'u32', + } + @property def dpp_ctrl_dialect(self) -> DppCtrlDialect: """DPP_CTRL naming and validity rules for this ISA.""" diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py index cdac71dfa85e..39137e7728ec 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py @@ -4126,7 +4126,7 @@ def generated_body(name: str, enc_name: str, *, result_writer=None) -> str: assert 'if (w > 0xFFFFFFFFULL) vcc |=' in carry -def test_rdna4_generator_wires_add3_clamp_from_encoded_vop3_field(): +def test_rdna4_generator_uses_instruction_policy_for_integer_clamp(): isa_xml = _mrisa_dir() / 'amdgpu_isa_rdna4.xml' parser = Parser(str(isa_xml), Rdna4Profile()) spec = parser.parse() @@ -4144,8 +4144,27 @@ def generated_body(name: str) -> str: ) add3 = generated_body('V_ADD3_U32') - assert 'vop3_integer_add3' in add3 - assert 'inst_.clamp' in add3 + assert 'vop3_integer_' not in add3 + assert 'inst_.clamp' not in add3 + + expected_helpers = { + 'V_MUL_I32_I24': 'vop3_integer_mul', + 'V_MUL_U32_U24': 'vop3_integer_mul', + 'V_MAD_I32_I24': 'vop3_integer_mad', + 'V_MAD_U32_U24': 'vop3_integer_mad', + 'V_MAD_I16': 'vop3_integer_mad', + 'V_MAD_U16': 'vop3_integer_mad', + 'V_MAD_I32_I16': 'vop3_integer_mad', + 'V_MAD_U32_U16': 'vop3_integer_mad', + 'V_SAD_U8': 'vop3_integer_sad_u8', + 'V_SAD_U16': 'vop3_integer_sad_u16', + 'V_SAD_U32': 'vop3_integer_sad_u32', + 'V_MSAD_U8': 'vop3_integer_msad_u8', + } + for name, helper in expected_helpers.items(): + body = generated_body(name) + assert helper in body + assert 'inst_.clamp' in body or3 = generated_body('V_OR3_B32') assert 'vop3_integer_add3' not in or3 diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py index 6ba971bbf37b..8edc1d93be61 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py @@ -1606,18 +1606,27 @@ def test_vop3_integer_clamp_saturates_before_narrowing(self, op, dtype, expected assert 'inst_.clamp' in cpp assert expected in cpp - def test_vop3_integer_add3_clamps_the_full_unsigned_sum(self): - sem = _FakeSem('V_ADD3_U32', 'vector_ternary', 'add3', 'u32') + @pytest.mark.parametrize( + 'name,op,dtype,helper', + [ + ('V_MUL_I32_I24', 'mul', 'i24', 'vop3_integer_mul'), + ('V_MUL_U32_U24', 'mul', 'u24', 'vop3_integer_mul'), + ('V_MAD_I32_I24', 'mad', 'i24', 'vop3_integer_mad'), + ('V_MAD_U32_U24', 'mad', 'u24', 'vop3_integer_mad'), + ], + ) + def test_vop3_integer_mul_mad_use_exact_saturating_intermediate( + self, name, op, dtype, helper + ): + sem_class = 'vector_binop' if op == 'mul' else 'vector_ternary' + sem = _FakeSem(name, sem_class, op, dtype) block = derive_sema_block(sem) cpp = lower_sema_block( block, - LoweringContext( - exec_model=block.pragma, - integer_saturation_dtype='u32', - ), + LoweringContext(exec_model=block.pragma, integer_saturation_dtype=dtype), ) - assert 'vop3_integer_add3' in cpp + assert helper in cpp assert 'inst_.clamp' in cpp def test_lshlrev(self): diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3_exec.cpp index 46a0696c9909..7f31c490e03a 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3_exec.cpp @@ -2279,7 +2279,8 @@ void VMadU32U16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } } @@ -2300,9 +2301,10 @@ void VMadI32I16Vop3::execute_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } } @@ -2793,7 +2795,7 @@ void VMadU16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -2807,10 +2809,7 @@ void VMadI16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3_exec.cpp index 282c6be18758..904a7de3ce4a 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3_exec.cpp @@ -2271,7 +2271,8 @@ void VMadU32U16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } } @@ -2292,9 +2293,10 @@ void VMadI32I16Vop3::execute_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } } @@ -2785,7 +2787,7 @@ void VMadU16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -2799,10 +2801,7 @@ void VMadI16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3_exec.cpp index 9bfb1ca7d05a..a8248f4df0c1 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3_exec.cpp @@ -2340,7 +2340,8 @@ void VMadU32U16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } } @@ -2361,9 +2362,10 @@ void VMadI32I16Vop3::execute_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } } @@ -2854,7 +2856,7 @@ void VMadU16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -2868,10 +2870,7 @@ void VMadI16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3_exec.cpp index c9e6a2d412b3..3615e3c83dc6 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3_exec.cpp @@ -2400,7 +2400,8 @@ void VMadU32U16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } } @@ -2421,9 +2422,10 @@ void VMadI32I16Vop3::execute_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } } @@ -2914,7 +2916,7 @@ void VMadU16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -2928,10 +2930,7 @@ void VMadI16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp index e1f4e8db228f..d92cc674bf32 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp @@ -1760,7 +1760,7 @@ void VMadU16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -1791,7 +1791,7 @@ RJ_NOINLINE void VMadU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -2478,10 +2478,7 @@ void VMadI16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -2512,10 +2509,7 @@ RJ_NOINLINE void VMadI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -2650,7 +2644,8 @@ void VMadU32U16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } } @@ -2683,7 +2678,8 @@ RJ_NOINLINE void VMadU32U16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } dpp_write_mask_scope_.restore(); } @@ -2709,9 +2705,10 @@ void VMadI32I16Vop3::execute_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } } @@ -2749,9 +2746,10 @@ RJ_NOINLINE void VMadI32I16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } dpp_write_mask_scope_.restore(); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop3_exec.cpp index 4a26b3c16d07..25afe8e7d161 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop3_exec.cpp @@ -6720,7 +6720,7 @@ void VMadU16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -6751,7 +6751,7 @@ RJ_NOINLINE void VMadU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -7926,10 +7926,7 @@ void VMadI16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -7960,10 +7957,7 @@ RJ_NOINLINE void VMadI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -8258,7 +8252,8 @@ void VMadU32U16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } } @@ -8291,7 +8286,8 @@ RJ_NOINLINE void VMadU32U16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } dpp_write_mask_scope_.restore(); } @@ -8317,9 +8313,10 @@ void VMadI32I16Vop3::execute_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } } @@ -8357,9 +8354,10 @@ RJ_NOINLINE void VMadI32I16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } dpp_write_mask_scope_.restore(); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop3_exec.cpp index cb6c9167f5b6..ae86828965e9 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop3_exec.cpp @@ -6720,7 +6720,7 @@ void VMadU16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -6751,7 +6751,7 @@ RJ_NOINLINE void VMadU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -7926,10 +7926,7 @@ void VMadI16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -7960,10 +7957,7 @@ RJ_NOINLINE void VMadI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -8258,7 +8252,8 @@ void VMadU32U16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } } @@ -8291,7 +8286,8 @@ RJ_NOINLINE void VMadU32U16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } dpp_write_mask_scope_.restore(); } @@ -8317,9 +8313,10 @@ void VMadI32I16Vop3::execute_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } } @@ -8357,9 +8354,10 @@ RJ_NOINLINE void VMadI32I16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } dpp_write_mask_scope_.restore(); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3_exec.cpp index 9053343509b8..d694022f3388 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3_exec.cpp @@ -7757,7 +7757,7 @@ void VMadU16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -7788,7 +7788,7 @@ RJ_NOINLINE void VMadU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -8477,10 +8477,7 @@ void VMadI16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -8511,10 +8508,7 @@ RJ_NOINLINE void VMadI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -8809,7 +8803,8 @@ void VMadU32U16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } } @@ -8842,7 +8837,8 @@ RJ_NOINLINE void VMadU32U16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } dpp_write_mask_scope_.restore(); } @@ -8868,9 +8864,10 @@ void VMadI32I16Vop3::execute_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } } @@ -8908,9 +8905,10 @@ RJ_NOINLINE void VMadI32I16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } dpp_write_mask_scope_.restore(); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h index 30ead02a0278..e254e7312f26 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h @@ -2884,12 +2884,10 @@ inline void execute_v_add3_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused] for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane( - inst, wf, inst.vdst, lane, - amdgpu::vop3_integer_add3(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), - amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane), - inst.inst_.clamp)); + sdwa::write_lane(inst, wf, inst.vdst, lane, + (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) + + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane) + + amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane))); } } @@ -13298,10 +13296,12 @@ inline void execute_v_mad_i16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] continue; sdwa::write_lane( inst, wf, inst.vdst, lane, - static_cast(static_cast(static_cast( - ((static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)) * - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))) + - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane))))))); + static_cast( + static_cast(static_cast(amdgpu::vop3_integer_mad( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane)), + inst.inst_.clamp))))); } } @@ -13323,9 +13323,10 @@ inline void execute_v_mad_i32_i16_vop3([[maybe_unused]] Inst &inst, int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane)); - sdwa::write_lane(inst, wf, inst.vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst.inst_.clamp)); } } @@ -13341,9 +13342,10 @@ inline void execute_v_mad_i32_i24_vop3([[maybe_unused]] Inst &inst, continue; sdwa::write_lane( inst, wf, inst.vdst, lane, - ::rocjitsu::amdgpu::mad_i24_u32(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), - amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane))); + amdgpu::vop3_integer_mad(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane), + inst.inst_.clamp)); } } @@ -13572,10 +13574,10 @@ inline void execute_v_mad_u16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] continue; sdwa::write_lane( inst, wf, inst.vdst, lane, - static_cast(static_cast(::rocjitsu::amdgpu::mad_lo_u16( + static_cast(static_cast(amdgpu::vop3_integer_mad( amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), - amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane))))); + amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane), inst.inst_.clamp)))); } } @@ -13592,7 +13594,8 @@ inline void execute_v_mad_u32_u16_vop3([[maybe_unused]] Inst &inst, uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(inst.src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(inst.src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane); - sdwa::write_lane(inst, wf, inst.vdst, lane, s0 * s1 + s2); + sdwa::write_lane(inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_mad(s0, s1, s2, inst.inst_.clamp)); } } @@ -13605,11 +13608,12 @@ inline void execute_v_mad_u32_u24_vop3([[maybe_unused]] Inst &inst, for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane( - inst, wf, inst.vdst, lane, - ((amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) & 0x00FFFFFFu) * - (amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane) & 0x00FFFFFFu) + - amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane))); + sdwa::write_lane(inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_mad( + amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane), + inst.inst_.clamp)); } } @@ -16279,20 +16283,12 @@ inline void execute_v_msad_u8_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { - auto a = amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane); - auto b = amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane); - auto c = amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane); - uint32_t r = c; - for (int i = 0; i < 4; ++i) { - uint8_t value = (a >> (i * 8)) & 0xFF; - uint8_t reference = (b >> (i * 8)) & 0xFF; - if (reference != 0) { - r += (value > reference) ? (value - reference) : (reference - value); - } - } - return r; - }()); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_msad_u8(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane), + inst.inst_.clamp)); } } @@ -16734,8 +16730,9 @@ inline void execute_v_mul_i32_i24_vop3([[maybe_unused]] Inst &inst, continue; sdwa::write_lane( inst, wf, inst.vdst, lane, - ::rocjitsu::amdgpu::mul_i24_u32(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); + amdgpu::vop3_integer_mul(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + inst.inst_.clamp)); } } @@ -16886,10 +16883,11 @@ inline void execute_v_mul_u32_u24_vop3([[maybe_unused]] Inst &inst, for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane( - inst, wf, inst.vdst, lane, - ((amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) & 0x00FFFFFFu) * - (amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane) & 0x00FFFFFFu))); + sdwa::write_lane(inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_mul( + amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + inst.inst_.clamp)); } } @@ -18602,16 +18600,12 @@ inline void execute_v_sad_u16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { - auto a = amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane); - auto b = amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane); - auto c = amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane); - uint16_t al = a & 0xFFFF; - uint16_t ah = (a >> 16) & 0xFFFF; - uint16_t bl = b & 0xFFFF; - uint16_t bh = (b >> 16) & 0xFFFF; - return c + ((al > bl) ? (al - bl) : (bl - al)) + ((ah > bh) ? (ah - bh) : (bh - ah)); - }()); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_sad_u16(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane), + inst.inst_.clamp)); } } @@ -18626,12 +18620,12 @@ inline void execute_v_sad_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { - auto a = amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane); - auto b = amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane); - auto c = amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane); - return c + ((a > b) ? (a - b) : (b - a)); - }()); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_sad_u32(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane), + inst.inst_.clamp)); } } @@ -18643,18 +18637,12 @@ inline void execute_v_sad_u8_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { - auto a = amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane); - auto b = amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane); - auto c = amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane); - uint32_t r = c; - for (int i = 0; i < 4; ++i) { - uint8_t ab = (a >> (i * 8)) & 0xFF; - uint8_t bb = (b >> (i * 8)) & 0xFF; - r += (ab > bb) ? (ab - bb) : (bb - ab); - } - return r; - }()); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_sad_u8(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane), + inst.inst_.clamp)); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h index c25bb8711e73..1b0366b45e76 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h @@ -119,13 +119,71 @@ inline std::make_unsigned_t vop3_integer_add(std::make_unsigned_t lhs, } } -/// @brief Execute unsigned VOP3 integer three-input addition, saturating when CLAMP is set. -template -inline std::make_unsigned_t vop3_integer_add3(std::make_unsigned_t lhs, - std::make_unsigned_t rhs, - std::make_unsigned_t addend, bool clamp) { - static_assert(std::is_integral_v && std::is_unsigned_v); - return vop3_integer_add(vop3_integer_add(lhs, rhs, clamp), addend, clamp); +/// @brief Execute an integer multiply-add with an exact intermediate. +template +inline std::make_unsigned_t vop3_integer_mad(uint32_t lhs, uint32_t rhs, + std::make_unsigned_t addend, bool clamp) { + static_assert(std::is_integral_v && sizeof(T) <= 4); + static_assert(SourceBits == 16 || SourceBits == 24); + using U = std::make_unsigned_t; + if constexpr (std::is_signed_v) { + const auto extend = [](uint32_t value) -> int64_t { + constexpr uint32_t shift = 32 - SourceBits; + return static_cast(static_cast(value << shift) >> shift); + }; + const int64_t wide = extend(lhs) * extend(rhs) + static_cast(std::bit_cast(addend)); + if (!clamp) + return static_cast(wide); + return static_cast(std::clamp(wide, static_cast(std::numeric_limits::min()), + static_cast(std::numeric_limits::max()))); + } else { + constexpr uint32_t source_mask = (uint32_t{1} << SourceBits) - 1u; + const uint64_t wide = static_cast(lhs & source_mask) * (rhs & source_mask) + addend; + if (clamp && wide > std::numeric_limits::max()) + return std::numeric_limits::max(); + return static_cast(wide); + } +} + +template +inline std::make_unsigned_t vop3_integer_mul(uint32_t lhs, uint32_t rhs, bool clamp) { + return vop3_integer_mad(lhs, rhs, std::make_unsigned_t{0}, clamp); +} + +inline uint32_t vop3_integer_sad_u8(uint32_t lhs, uint32_t rhs, uint32_t addend, bool clamp) { + uint32_t difference = 0; + for (unsigned i = 0; i < 4; ++i) { + const uint32_t a = (lhs >> (i * 8)) & 0xffu; + const uint32_t b = (rhs >> (i * 8)) & 0xffu; + difference += a > b ? a - b : b - a; + } + return vop3_integer_add(difference, addend, clamp); +} + +inline uint32_t vop3_integer_sad_u16(uint32_t lhs, uint32_t rhs, uint32_t addend, bool clamp) { + uint32_t difference = 0; + for (unsigned i = 0; i < 2; ++i) { + const uint32_t a = (lhs >> (i * 16)) & 0xffffu; + const uint32_t b = (rhs >> (i * 16)) & 0xffffu; + difference += a > b ? a - b : b - a; + } + return vop3_integer_add(difference, addend, clamp); +} + +inline uint32_t vop3_integer_sad_u32(uint32_t lhs, uint32_t rhs, uint32_t addend, bool clamp) { + const uint32_t difference = lhs > rhs ? lhs - rhs : rhs - lhs; + return vop3_integer_add(difference, addend, clamp); +} + +inline uint32_t vop3_integer_msad_u8(uint32_t lhs, uint32_t rhs, uint32_t addend, bool clamp) { + uint32_t difference = 0; + for (unsigned i = 0; i < 4; ++i) { + const uint32_t a = (lhs >> (i * 8)) & 0xffu; + const uint32_t b = (rhs >> (i * 8)) & 0xffu; + if (b != 0) + difference += a > b ? a - b : b - a; + } + return vop3_integer_add(difference, addend, clamp); } /// @brief Execute VOP3 integer subtraction, saturating when CLAMP is set. @@ -2334,8 +2392,9 @@ template [[nodiscard]] inline bool try_execute_ternary_vop3_true16_src01_simd(Inst &inst, Wavefront &wf, TernOp tern_op) { static_assert(std::is_same_v); - if (simd_force_scalar() || !sdwa::supports_direct_simd_store(inst) || !inst.src0.simd_capable() || - !inst.src1.simd_capable() || !inst.src2.simd_capable() || !inst.vdst.simd_capable()) + if (inst.inst_.clamp != 0u || simd_force_scalar() || !sdwa::supports_direct_simd_store(inst) || + !inst.src0.simd_capable() || !inst.src1.simd_capable() || !inst.src2.simd_capable() || + !inst.vdst.simd_capable()) return false; constexpr std::size_t W = util::native_width_v; const uint64_t chunk_full = util::mask(static_cast(W)); diff --git a/emulation/rocjitsu/tests/cdna5_execution_test.cpp b/emulation/rocjitsu/tests/cdna5_execution_test.cpp index d1c9b298f67d..c5176c2904c6 100644 --- a/emulation/rocjitsu/tests/cdna5_execution_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_execution_test.cpp @@ -228,6 +228,58 @@ TEST(Gfx1250ExecutionTest, Vop3IntegerClampSaturatesBeforeNarrowing) { } } +TEST(Gfx1250ExecutionTest, Vop3IntegerMultiplyAccumulateClampSaturatesExactResult) { + ForceScalarGuard guard; + struct TestCase { + uint16_t opcode; + uint32_t src0; + uint32_t src1; + uint32_t src2; + uint32_t expected; + }; + constexpr std::array kCases{ + TestCase{cdna5::kVMulU32U24Vop3, 0x00ffffffu, 0x00ffffffu, 0u, UINT32_MAX}, + TestCase{cdna5::kVMulI32I24Vop3, 0x007fffffu, 0x007fffffu, 0u, + static_cast(INT32_MAX)}, + TestCase{cdna5::kVMadU32U24Vop3, 0x00ffffffu, 0x00ffffffu, UINT32_MAX, UINT32_MAX}, + TestCase{cdna5::kVMadI32I24Vop3, 0x007fffffu, 0x007fffffu, static_cast(INT32_MAX), + static_cast(INT32_MAX)}, + TestCase{cdna5::kVMadU16Vop3, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX}, + TestCase{cdna5::kVMadI16Vop3, static_cast(INT16_MAX), + static_cast(INT16_MAX), static_cast(INT16_MAX), + static_cast(INT16_MAX)}, + TestCase{cdna5::kVMadU32U16Vop3, UINT16_MAX, UINT16_MAX, UINT32_MAX, UINT32_MAX}, + TestCase{cdna5::kVMadI32I16Vop3, static_cast(INT16_MAX), + static_cast(INT16_MAX), static_cast(INT32_MAX), + static_cast(INT32_MAX)}, + TestCase{cdna5::kVSadU32Vop3, 0u, UINT32_MAX, 1u, UINT32_MAX}, + TestCase{cdna5::kVMsadU8Vop3, 0xffffffffu, 0x01010101u, UINT32_MAX, UINT32_MAX}, + }; + + for (uint32_t scalar = 0; scalar < 2; ++scalar) { + util::set_force_scalar_for_testing(scalar != 0); + for (const TestCase &test : kCases) { + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + const uint32_t base = wf->vgpr_alloc().base; + cu->write_vgpr(base, 0, test.src0); + cu->write_vgpr(base + 1, 0, test.src1); + cu->write_vgpr(base + 2, 0, test.src2); + const auto words = cdna5::build_vop3( + test.opcode, {.vdst = 3, .clamp = 1, .src0 = 256, .src1 = 257, .src2 = 258}); + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA5); + ASSERT_NE(decoder, nullptr); + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + inst->execute(*inst, wf); + EXPECT_EQ(cu->read_vgpr(base + 3, 0), test.expected) << "scalar " << scalar; + } + } +} + TEST(Gfx1250ExecutionTest, Vop3U64ClampSaturatesNormalAndDppPaths) { constexpr uint32_t kSrc0 = 0; constexpr uint32_t kSrc1 = 2; diff --git a/emulation/rocjitsu/tests/simd_correctness/vop3_ternary_int_simd_correctness_test.cpp b/emulation/rocjitsu/tests/simd_correctness/vop3_ternary_int_simd_correctness_test.cpp index 1f3ee2db47cc..0ff72c05b6b4 100644 --- a/emulation/rocjitsu/tests/simd_correctness/vop3_ternary_int_simd_correctness_test.cpp +++ b/emulation/rocjitsu/tests/simd_correctness/vop3_ternary_int_simd_correctness_test.cpp @@ -219,65 +219,6 @@ TEST(Vop3TernaryIntSimdCorrectness, PartialExec) { check_case(c, /*exec=*/0xA5A5'F0F0'1234'8001ULL); } -TEST(Vop3TernaryIntSimdCorrectness, Rdna4Add3ClampSaturatesNormalAndDppPaths) { - ForceScalarGuard gate_guard; - ScopedIsaExecutionBackend execution_backend_scope{&rdna4::execution_backend()}; - constexpr uint32_t kSrc0 = 0; - constexpr uint32_t kSrc1 = 1; - constexpr uint32_t kSrc2 = 2; - - auto run_case = [&](bool force_scalar, bool dpp, bool clamp) { - util::set_force_scalar_for_testing(force_scalar); - Fixture fx(ROCJITSU_CODE_ARCH_RDNA4); - EXPECT_NE(fx.cu, nullptr); - EXPECT_NE(fx.wf, nullptr); - if (fx.cu == nullptr || fx.wf == nullptr) - return 0u; - const uint32_t base = fx.wf->vgpr_alloc().base; - fx.wf->set_exec(1u); - fx.cu->write_vgpr(base + kSrc0, 0, UINT32_MAX); - fx.cu->write_vgpr(base + kSrc1, 0, 1u); - fx.cu->write_vgpr(base + kSrc2, 0, 0u); - fx.cu->write_vgpr(base + kDstVgpr, 0, DST_SENTINEL); - - if (!dpp) { - const auto words = - rdna4::build_vop3(rdna4::kVAdd3U32Vop3, {.vdst = kDstVgpr, - .clamp = static_cast(clamp), - .src0 = 256 + kSrc0, - .src1 = 256 + kSrc1, - .src2 = 256 + kSrc2}); - std::unique_ptr inst(decode_valid(*fx.decoder, words.data())); - EXPECT_NE(inst, nullptr); - if (inst == nullptr) - return 0u; - fx.cu->execute_instruction(inst.get(), *fx.wf); - } else { - rdna4::Vop3VopDpp16MachineInst raw{}; - raw.vdst = kDstVgpr; - raw.clamp = clamp; - raw.src0 = amdgpu::SRC_DPP; - raw.src1 = 256 + kSrc1; - raw.src2 = 256 + kSrc2; - raw.vsrc0 = kSrc0; - raw.dpp_ctrl = amdgpu::dpp::ROW_SELECT_BASE; - raw.bound_ctrl = 1; - raw.bank_mask = 0xF; - raw.row_mask = 0xF; - rdna4::VAdd3U32Vop3 inst(reinterpret_cast(&raw)); - inst.execute_impl(*fx.wf); - } - - return fx.cu->read_vgpr(base + kDstVgpr, 0); - }; - - for (bool force_scalar : {false, true}) { - EXPECT_EQ(run_case(force_scalar, false, true), UINT32_MAX); - EXPECT_EQ(run_case(force_scalar, true, true), UINT32_MAX); - EXPECT_EQ(run_case(force_scalar, false, false), 0u); - } -} - TEST(Vop3TernaryIntSimdCorrectness, IsaGoldenByteOperations) { struct GoldenCase { const char *name; From 0b6f0694e3b4a0267894139088fe1f3aa32d5daa Mon Sep 17 00:00:00 2001 From: Sambhav Jain Date: Wed, 2 Sep 2026 19:32:58 +0000 Subject: [PATCH 06/16] [rocjitsu] Complete integer clamp coverage Handle the assembler-legal V_SAD_HI_U8 clamp form with an exact shifted accumulator before saturation. Extend execution coverage across signed lower bounds and unclamped I16/I24 wraparound behavior. Signed-off-by: Sambhav Jain --- .../amdisa/codegen/execute/sema_lower.py | 1 + .../rocjitsu/lib/python/amdisa/isa_profile.py | 1 + .../tests/test_generator_profile_gates.py | 1 + .../amdgpu/generated/shared/execute_shared.h | 18 ++++++------------ .../isa/arch/amdgpu/shared/simd_glue.h | 14 ++++++++++++++ .../rocjitsu/tests/cdna5_execution_test.cpp | 19 +++++++++++++++++-- 6 files changed, 40 insertions(+), 14 deletions(-) diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py index 76d972ea77e4..e6a218410cfc 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py @@ -1778,6 +1778,7 @@ def _lower_call(node: SemaNode, ctx: LoweringContext) -> str: ) if ctx.integer_saturation_dtype == 'u32' and callee in ( + 'sad_hi_u8', 'sad_u8', 'sad_u16', 'sad_u32', diff --git a/emulation/rocjitsu/lib/python/amdisa/isa_profile.py b/emulation/rocjitsu/lib/python/amdisa/isa_profile.py index dc3771e1325e..f025612fe123 100644 --- a/emulation/rocjitsu/lib/python/amdisa/isa_profile.py +++ b/emulation/rocjitsu/lib/python/amdisa/isa_profile.py @@ -610,6 +610,7 @@ def integer_clamp_dtypes(self) -> dict[str, str]: 'V_MSAD_U8': 'u32', 'V_MUL_I32_I24': 'i32', 'V_MUL_U32_U24': 'u32', + 'V_SAD_HI_U8': 'u32', 'V_SAD_U8': 'u32', 'V_SAD_U16': 'u32', 'V_SAD_U32': 'u32', diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py index 39137e7728ec..70d7580d0e6a 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py @@ -4156,6 +4156,7 @@ def generated_body(name: str) -> str: 'V_MAD_U16': 'vop3_integer_mad', 'V_MAD_I32_I16': 'vop3_integer_mad', 'V_MAD_U32_U16': 'vop3_integer_mad', + 'V_SAD_HI_U8': 'vop3_integer_sad_hi_u8', 'V_SAD_U8': 'vop3_integer_sad_u8', 'V_SAD_U16': 'vop3_integer_sad_u16', 'V_SAD_U32': 'vop3_integer_sad_u32', diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h index e254e7312f26..6f813e218fe8 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h @@ -18569,18 +18569,12 @@ inline void execute_v_sad_hi_u8_vop3([[maybe_unused]] Inst &inst, [[maybe_unused for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { - auto a = amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane); - auto b = amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane); - auto c = amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane); - uint32_t r = 0; - for (int i = 0; i < 4; ++i) { - uint8_t ab = (a >> (i * 8)) & 0xFF; - uint8_t bb = (b >> (i * 8)) & 0xFF; - r += (ab > bb) ? (ab - bb) : (bb - ab); - } - return (r << 16) + c; - }()); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_sad_hi_u8(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane), + inst.inst_.clamp)); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h index 1b0366b45e76..f4c799b59011 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h @@ -160,6 +160,20 @@ inline uint32_t vop3_integer_sad_u8(uint32_t lhs, uint32_t rhs, uint32_t addend, return vop3_integer_add(difference, addend, clamp); } +/// @brief Execute shifted byte SAD plus accumulation, saturating when CLAMP is set. +inline uint32_t vop3_integer_sad_hi_u8(uint32_t lhs, uint32_t rhs, uint32_t addend, bool clamp) { + uint32_t difference = 0; + for (unsigned i = 0; i < 4; ++i) { + const uint32_t a = (lhs >> (i * 8)) & 0xffu; + const uint32_t b = (rhs >> (i * 8)) & 0xffu; + difference += a > b ? a - b : b - a; + } + const uint64_t wide = (static_cast(difference) << 16) + addend; + if (clamp && wide > UINT32_MAX) + return UINT32_MAX; + return static_cast(wide); +} + inline uint32_t vop3_integer_sad_u16(uint32_t lhs, uint32_t rhs, uint32_t addend, bool clamp) { uint32_t difference = 0; for (unsigned i = 0; i < 2; ++i) { diff --git a/emulation/rocjitsu/tests/cdna5_execution_test.cpp b/emulation/rocjitsu/tests/cdna5_execution_test.cpp index c5176c2904c6..f24e6b42c5cf 100644 --- a/emulation/rocjitsu/tests/cdna5_execution_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_execution_test.cpp @@ -236,22 +236,34 @@ TEST(Gfx1250ExecutionTest, Vop3IntegerMultiplyAccumulateClampSaturatesExactResul uint32_t src1; uint32_t src2; uint32_t expected; + bool clamp = true; }; constexpr std::array kCases{ TestCase{cdna5::kVMulU32U24Vop3, 0x00ffffffu, 0x00ffffffu, 0u, UINT32_MAX}, TestCase{cdna5::kVMulI32I24Vop3, 0x007fffffu, 0x007fffffu, 0u, static_cast(INT32_MAX)}, + TestCase{cdna5::kVMulI32I24Vop3, 0x00800000u, 257u, 0u, static_cast(INT32_MIN)}, TestCase{cdna5::kVMadU32U24Vop3, 0x00ffffffu, 0x00ffffffu, UINT32_MAX, UINT32_MAX}, TestCase{cdna5::kVMadI32I24Vop3, 0x007fffffu, 0x007fffffu, static_cast(INT32_MAX), static_cast(INT32_MAX)}, + TestCase{cdna5::kVMadI32I24Vop3, 0x00800000u, 256u, UINT32_MAX, + static_cast(INT32_MIN)}, + TestCase{cdna5::kVMadI32I24Vop3, 0x00800000u, 256u, UINT32_MAX, + static_cast(INT32_MAX), false}, TestCase{cdna5::kVMadU16Vop3, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX}, TestCase{cdna5::kVMadI16Vop3, static_cast(INT16_MAX), static_cast(INT16_MAX), static_cast(INT16_MAX), static_cast(INT16_MAX)}, + TestCase{cdna5::kVMadI16Vop3, static_cast(INT16_MIN), 2u, static_cast(-1), + static_cast(INT16_MIN)}, TestCase{cdna5::kVMadU32U16Vop3, UINT16_MAX, UINT16_MAX, UINT32_MAX, UINT32_MAX}, TestCase{cdna5::kVMadI32I16Vop3, static_cast(INT16_MAX), static_cast(INT16_MAX), static_cast(INT32_MAX), static_cast(INT32_MAX)}, + TestCase{cdna5::kVMadI32I16Vop3, static_cast(INT16_MIN), + static_cast(INT16_MAX), static_cast(INT32_MIN), + static_cast(INT32_MIN)}, + TestCase{cdna5::kVSadHiU8Vop3, UINT32_MAX, 0u, 0xfc040000u, UINT32_MAX}, TestCase{cdna5::kVSadU32Vop3, 0u, UINT32_MAX, 1u, UINT32_MAX}, TestCase{cdna5::kVMsadU8Vop3, 0xffffffffu, 0x01010101u, UINT32_MAX, UINT32_MAX}, }; @@ -268,8 +280,11 @@ TEST(Gfx1250ExecutionTest, Vop3IntegerMultiplyAccumulateClampSaturatesExactResul cu->write_vgpr(base, 0, test.src0); cu->write_vgpr(base + 1, 0, test.src1); cu->write_vgpr(base + 2, 0, test.src2); - const auto words = cdna5::build_vop3( - test.opcode, {.vdst = 3, .clamp = 1, .src0 = 256, .src1 = 257, .src2 = 258}); + const auto words = cdna5::build_vop3(test.opcode, {.vdst = 3, + .clamp = static_cast(test.clamp), + .src0 = 256, + .src1 = 257, + .src2 = 258}); auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA5); ASSERT_NE(decoder, nullptr); std::unique_ptr inst(decode_valid(*decoder, words.data())); From 134267a7c677e03d4b929a54119f0cf6611fc66f Mon Sep 17 00:00:00 2001 From: Sambhav Jain Date: Wed, 2 Sep 2026 19:48:38 +0000 Subject: [PATCH 07/16] [rocjitsu] Cover cleared carry and borrow results Seed the destination status bit before in-range clamped add and subtract cases. Verify both the arithmetic result and clearing of stale carry or borrow state in scalar and SIMD-selected execution. Signed-off-by: Sambhav Jain --- .../rocjitsu/tests/cdna5_execution_test.cpp | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/emulation/rocjitsu/tests/cdna5_execution_test.cpp b/emulation/rocjitsu/tests/cdna5_execution_test.cpp index f24e6b42c5cf..f31ce3b04219 100644 --- a/emulation/rocjitsu/tests/cdna5_execution_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_execution_test.cpp @@ -402,14 +402,19 @@ TEST(Gfx1250ExecutionTest, Vop3CarryClampSaturatesVdstAndPreservesCarryBorrow) { uint32_t src0; uint32_t src1; bool carry_in; + uint32_t expected_vdst; + bool expected_carry; + bool initial_carry = false; }; constexpr std::array kCases{ - TestCase{cdna5::kVAddCoU32Vop3SdstEnc, UINT32_MAX, 1u, false}, - TestCase{cdna5::kVSubCoU32Vop3SdstEnc, 0u, 1u, false}, - TestCase{cdna5::kVSubrevCoU32Vop3SdstEnc, 1u, 0u, false}, - TestCase{cdna5::kVAddCoCiU32Vop3SdstEnc, UINT32_MAX, 0u, true}, - TestCase{cdna5::kVSubCoCiU32Vop3SdstEnc, 0u, 0u, true}, - TestCase{cdna5::kVSubrevCoCiU32Vop3SdstEnc, 0u, 0u, true}, + TestCase{cdna5::kVAddCoU32Vop3SdstEnc, UINT32_MAX, 1u, false, UINT32_MAX, true}, + TestCase{cdna5::kVSubCoU32Vop3SdstEnc, 0u, 1u, false, 0u, true}, + TestCase{cdna5::kVSubrevCoU32Vop3SdstEnc, 1u, 0u, false, 0u, true}, + TestCase{cdna5::kVAddCoCiU32Vop3SdstEnc, UINT32_MAX, 0u, true, UINT32_MAX, true}, + TestCase{cdna5::kVSubCoCiU32Vop3SdstEnc, 0u, 0u, true, 0u, true}, + TestCase{cdna5::kVSubrevCoCiU32Vop3SdstEnc, 0u, 0u, true, 0u, true}, + TestCase{cdna5::kVAddCoU32Vop3SdstEnc, 1u, 2u, false, 3u, false, true}, + TestCase{cdna5::kVSubCoU32Vop3SdstEnc, 2u, 1u, false, 1u, false, true}, }; for (uint32_t scalar = 0; scalar < 2; ++scalar) { util::set_force_scalar_for_testing(scalar != 0); @@ -423,7 +428,7 @@ TEST(Gfx1250ExecutionTest, Vop3CarryClampSaturatesVdstAndPreservesCarryBorrow) { cu->write_vgpr(base, 0, test.src0); cu->write_vgpr(base + 1, 0, test.src1); write_wave_sgpr(*cu, *wf, 4, test.carry_in ? 1u : 0u); - write_wave_sgpr(*cu, *wf, 2, 0u); + write_wave_sgpr(*cu, *wf, 2, test.initial_carry ? 1u : 0u); const auto words = cdna5::build_vop3_sdst_enc( test.opcode, {.vdst = 2, .sdst = 2, .clamp = 1, .src0 = 256, .src1 = 257, .src2 = 4}); auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA5); @@ -431,12 +436,9 @@ TEST(Gfx1250ExecutionTest, Vop3CarryClampSaturatesVdstAndPreservesCarryBorrow) { std::unique_ptr inst(decode_valid(*decoder, words.data())); ASSERT_NE(inst, nullptr); inst->execute(*inst, wf); - EXPECT_EQ(cu->read_vgpr(base + 2, 0), test.opcode == cdna5::kVAddCoU32Vop3SdstEnc || - test.opcode == cdna5::kVAddCoCiU32Vop3SdstEnc - ? UINT32_MAX - : 0u) + EXPECT_EQ(cu->read_vgpr(base + 2, 0), test.expected_vdst) << "scalar " << scalar; + EXPECT_EQ(read_wave_sgpr(*cu, *wf, 2) & 1u, test.expected_carry ? 1u : 0u) << "scalar " << scalar; - EXPECT_EQ(read_wave_sgpr(*cu, *wf, 2) & 1u, 1u) << "scalar " << scalar; } } } From 0ffdc8c639b411f91b83ee6a1230f00028411f8d Mon Sep 17 00:00:00 2001 From: Sambhav Jain Date: Wed, 2 Sep 2026 20:45:40 +0000 Subject: [PATCH 08/16] [rocjitsu] Complete remaining integer clamp families Saturate ADD_MIN/MAX before selecting the third operand and honor CLAMP for packed and full-width QSAD accumulation. Add generator and execution coverage for the remaining legal instruction families. Signed-off-by: Sambhav Jain --- .../python/amdisa/codegen/execute/__init__.py | 6 +- .../amdisa/codegen/execute/sema_lower.py | 13 ++ .../amdisa/codegen/execute/vector_special.py | 20 +++- .../rocjitsu/lib/python/amdisa/isa_profile.py | 7 ++ .../tests/test_generator_profile_gates.py | 15 +++ .../python/amdisa/tests/test_sema_derive.py | 17 ++- .../tests/test_vector_special_codegen.py | 8 +- .../arch/amdgpu/generated/cdna1/vop3_exec.cpp | 12 +- .../arch/amdgpu/generated/cdna2/vop3_exec.cpp | 12 +- .../arch/amdgpu/generated/cdna3/vop3_exec.cpp | 12 +- .../arch/amdgpu/generated/cdna4/vop3_exec.cpp | 12 +- .../amdgpu/generated/cdna5/vop3_exec_misc.cpp | 12 +- .../generated/cdna5/vop3_exec_ternary.cpp | 112 ++++++++---------- .../arch/amdgpu/generated/rdna1/vop3_exec.cpp | 12 +- .../arch/amdgpu/generated/rdna2/vop3_exec.cpp | 12 +- .../arch/amdgpu/generated/rdna3/vop3_exec.cpp | 12 +- .../amdgpu/generated/rdna3_5/vop3_exec.cpp | 12 +- .../arch/amdgpu/generated/rdna4/vop3_exec.cpp | 12 +- .../isa/arch/amdgpu/shared/simd_glue.h | 17 +++ .../rocjitsu/tests/cdna5_execution_test.cpp | 41 +++++++ 20 files changed, 269 insertions(+), 107 deletions(-) diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py index 82f84476971b..ceb6461a10ef 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py @@ -208,7 +208,11 @@ def _register_handlers() -> None: c.dst_ops, c.src_ops, c.op, c.profile.uses_vgpr_msb_indexing ) DISPATCH['vector_qsad'] = lambda c: gen_vector_qsad( - c.dst_ops, c.src_ops, c.op, c.profile.uses_vgpr_msb_indexing + c.dst_ops, + c.src_ops, + c.op, + c.profile.uses_vgpr_msb_indexing, + c.is_vop3 and c.inst.name in c.profile.integer_clamp_dtypes, ) DISPATCH['vector_trig_preop'] = lambda c: gen_vector_trig_preop( c.dst_ops, c.src_ops, c.is_vop3, c.has_abs diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py index e6a218410cfc..a6bb99ef6b2c 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py @@ -1763,6 +1763,19 @@ def _lower_call(node: SemaNode, ctx: LoweringContext) -> str: args = [_lower_expr(c, ctx) for c in node.children[1:]] args_str = ', '.join(args) + add_minmax_calls = { + 'add_max_i32': ('int32_t', 'true'), + 'add_max_u32': ('uint32_t', 'true'), + 'add_min_i32': ('int32_t', 'false'), + 'add_min_u32': ('uint32_t', 'false'), + } + if ctx.integer_saturation_dtype is not None and callee in add_minmax_calls: + cpp_type, select_max = add_minmax_calls[callee] + return ( + f'amdgpu::vop3_integer_add_minmax<{cpp_type}, {select_max}>(' + f'{args_str}, inst_.clamp)' + ) + saturating_calls = { 'mul_i24': ('int32_t', 24, False), 'mul_u24': ('uint32_t', 24, False), diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py index 8e14d8d4ddaf..4906b8bfb68d 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py @@ -496,7 +496,11 @@ def gen_vector_perm_pk16( def gen_vector_qsad( - dst: list[str], src: list[str], op: str | None, uses_vgpr_msb_indexing: bool + dst: list[str], + src: list[str], + op: str | None, + uses_vgpr_msb_indexing: bool, + integer_clamp: bool = False, ) -> str: """Generate QSAD/MQSAD four-window sum-of-absolute-difference ops.""" is_packed = op in ('sad_pk_u16', 'msad_pk_u16') @@ -553,17 +557,27 @@ def gen_vector_qsad( L.append(' sum += a > b ? a - b : b - a;') L.extend([' }']) if is_packed: + value_expr = ( + 'inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)' + if integer_clamp + else 'value & 0xffffu' + ) L.extend( [ ' uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu);', - ' packed_result |= static_cast(value & 0xffffu) << (window * 16);', + f' packed_result |= static_cast({value_expr}) << (window * 16);', ] ) else: + value_expr = ( + 'amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)' + if integer_clamp + else 'accum[window] + sum' + ) L.extend( [ f' dst_word.encoding_value_ = {dst[0]}.encoding_value_ + static_cast(window);', - ' amdgpu::RegisterAccess(wf).write_lane(dst_word, lane, accum[window] + sum);', + f' amdgpu::RegisterAccess(wf).write_lane(dst_word, lane, {value_expr});', ] ) L.append(' }') diff --git a/emulation/rocjitsu/lib/python/amdisa/isa_profile.py b/emulation/rocjitsu/lib/python/amdisa/isa_profile.py index f025612fe123..e827ecd2fe5d 100644 --- a/emulation/rocjitsu/lib/python/amdisa/isa_profile.py +++ b/emulation/rocjitsu/lib/python/amdisa/isa_profile.py @@ -590,6 +590,10 @@ def integer_clamp_dtypes(self) -> dict[str, str]: """ return { 'V_ADDC_CO_U32': 'u32', + 'V_ADD_MAX_I32': 'i32', + 'V_ADD_MAX_U32': 'u32', + 'V_ADD_MIN_I32': 'i32', + 'V_ADD_MIN_U32': 'u32', 'V_ADD_CO_CI_U32': 'u32', 'V_ADD_CO_U32': 'u32', 'V_ADD_I16': 'i16', @@ -607,6 +611,8 @@ def integer_clamp_dtypes(self) -> dict[str, str]: 'V_MAD_U16': 'u16', 'V_MAD_U32_U16': 'u32', 'V_MAD_U32_U24': 'u32', + 'V_MQSAD_PK_U16_U8': 'u16', + 'V_MQSAD_U32_U8': 'u32', 'V_MSAD_U8': 'u32', 'V_MUL_I32_I24': 'i32', 'V_MUL_U32_U24': 'u32', @@ -614,6 +620,7 @@ def integer_clamp_dtypes(self) -> dict[str, str]: 'V_SAD_U8': 'u32', 'V_SAD_U16': 'u32', 'V_SAD_U32': 'u32', + 'V_QSAD_PK_U16_U8': 'u16', 'V_SUBBREV_CO_U32': 'u32', 'V_SUBB_CO_U32': 'u32', 'V_SUBREV_CO_CI_U32': 'u32', diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py index 70d7580d0e6a..3a62d0e1b263 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py @@ -4114,6 +4114,21 @@ def generated_body(name: str, enc_name: str, *, result_writer=None) -> str: assert 'vop3_integer_sub' in sub_u64 assert 'inst_.clamp' in sub_u64 + expected_helpers = { + 'V_ADD_MAX_I32': 'vop3_integer_add_minmax', + 'V_ADD_MAX_U32': 'vop3_integer_add_minmax', + 'V_ADD_MIN_I32': 'vop3_integer_add_minmax', + 'V_ADD_MIN_U32': 'vop3_integer_add_minmax', + } + for name, helper in expected_helpers.items(): + body = generated_body(name, 'ENC_VOP3') + assert helper in body + assert 'inst_.clamp' in body + + for name in ('V_QSAD_PK_U16_U8', 'V_MQSAD_PK_U16_U8', 'V_MQSAD_U32_U8'): + body = generated_body(name, 'ENC_VOP3') + assert 'inst_.clamp' in body + vop2 = generated_body('V_ADD_NC_U32', 'ENC_VOP2') assert 'vop3_integer_add' not in vop2 assert 'inst_.clamp' not in vop2 diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py index 8edc1d93be61..cf931d5fbb49 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py @@ -1705,7 +1705,7 @@ def test_lowers_to_std_fma(self): cpp = lower_sema_block(block) assert 'std::fma(' in cpp - def test_add_minmax_i32_u32_wraps_add_before_clamp(self): + def test_add_minmax_i32_u32_can_saturate_add_before_selection(self): cases = [ ('V_ADD_MAX_I32', 'add_max', 'i32', 'std::max'), ('V_ADD_MAX_U32', 'add_max', 'u32', 'std::max'), @@ -1720,10 +1720,17 @@ def test_add_minmax_i32_u32_wraps_add_before_clamp(self): assert sem.data_type == dtype block = derive_sema_block(sem) - cpp = lower_sema_block(block) - assert clamp_fn in cpp - assert 'static_cast' in cpp - assert f'{op}_{dtype}(' not in cpp + cpp = lower_sema_block( + block, + LoweringContext( + exec_model=ExecModel.VECTOR, + integer_saturation_dtype=dtype, + ), + ) + select_max = 'true' if clamp_fn == 'std::max' else 'false' + assert f'vop3_integer_add_minmax<' in cpp + assert f', {select_max}>' in cpp + assert 'inst_.clamp' in cpp def test_ashr_pk_i8_u8_i32_packs_shifted_saturated_bytes(self): cases = [ diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_vector_special_codegen.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_vector_special_codegen.py index b5f242b82a34..236b26aaab04 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_vector_special_codegen.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_vector_special_codegen.py @@ -112,8 +112,8 @@ def test_saturating_pack_codegen_covers_all_narrowing_modes(): def test_perm_and_qsad_codegen_cover_multiword_results(): perm = gen_vector_perm_pk16(['vdst'], ['src0', 'src1', 'src2'], 'b6', True) - qsad = gen_vector_qsad(['vdst'], ['src0', 'src1', 'src2'], 'sad_pk_u16', True) - mqsad = gen_vector_qsad(['vdst'], ['src0', 'src1', 'src2'], 'msad_u32', True) + qsad = gen_vector_qsad(['vdst'], ['src0', 'src1', 'src2'], 'sad_pk_u16', True, True) + mqsad = gen_vector_qsad(['vdst'], ['src0', 'src1', 'src2'], 'msad_u32', True, True) assert 'uint32_t packed[3]' in perm assert 'source_bit = index * 6u' in perm @@ -122,11 +122,11 @@ def test_perm_and_qsad_codegen_cover_multiword_results(): assert 'read_lane64(src1, lane)' in perm assert '(window + byte) * 8' in qsad assert 'write_lane64(vdst, lane, packed_result)' in qsad - assert 'value & 0xffffu' in qsad + assert 'inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)' in qsad assert 'if (b != 0)' in mqsad assert 'accum[window] = amdgpu::RegisterAccess(wf).read_lane' in mqsad assert 'if (std::optional constant = src2.const_value())' in mqsad - assert 'accum[window] + sum' in mqsad + assert 'vop3_integer_add(accum[window], sum, inst_.clamp)' in mqsad @pytest.mark.parametrize( diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3_exec.cpp index 7f31c490e03a..98fe898af04f 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3_exec.cpp @@ -1995,7 +1995,9 @@ void VQsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -2019,7 +2021,9 @@ void VMqsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -2057,7 +2061,9 @@ void VMqsadU32U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } dst_word.encoding_value_ = vdst.encoding_value_ + static_cast(window); - amdgpu::sdwa::write_lane(*this, wf, dst_word, lane, accum[window] + sum); + amdgpu::sdwa::write_lane( + *this, wf, dst_word, lane, + amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)); } } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3_exec.cpp index 904a7de3ce4a..160c2c351b3d 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3_exec.cpp @@ -1987,7 +1987,9 @@ void VQsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -2011,7 +2013,9 @@ void VMqsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -2049,7 +2053,9 @@ void VMqsadU32U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } dst_word.encoding_value_ = vdst.encoding_value_ + static_cast(window); - amdgpu::sdwa::write_lane(*this, wf, dst_word, lane, accum[window] + sum); + amdgpu::sdwa::write_lane( + *this, wf, dst_word, lane, + amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)); } } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3_exec.cpp index a8248f4df0c1..fe330f7b857d 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3_exec.cpp @@ -2056,7 +2056,9 @@ void VQsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -2080,7 +2082,9 @@ void VMqsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -2118,7 +2122,9 @@ void VMqsadU32U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } dst_word.encoding_value_ = vdst.encoding_value_ + static_cast(window); - amdgpu::sdwa::write_lane(*this, wf, dst_word, lane, accum[window] + sum); + amdgpu::sdwa::write_lane( + *this, wf, dst_word, lane, + amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)); } } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3_exec.cpp index 3615e3c83dc6..527d37f95a4b 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3_exec.cpp @@ -2116,7 +2116,9 @@ void VQsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -2140,7 +2142,9 @@ void VMqsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -2178,7 +2182,9 @@ void VMqsadU32U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } dst_word.encoding_value_ = vdst.encoding_value_ + static_cast(window); - amdgpu::sdwa::write_lane(*this, wf, dst_word, lane, accum[window] + sum); + amdgpu::sdwa::write_lane( + *this, wf, dst_word, lane, + amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)); } } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_misc.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_misc.cpp index 4deec9eca8b6..a5f90f58d1d6 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_misc.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_misc.cpp @@ -247,7 +247,9 @@ void VQsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -271,7 +273,9 @@ void VMqsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -309,7 +313,9 @@ void VMqsadU32U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } dst_word.encoding_value_ = vdst.encoding_value_ + static_cast(window); - amdgpu::sdwa::write_lane(*this, wf, dst_word, lane, accum[window] + sum); + amdgpu::sdwa::write_lane( + *this, wf, dst_word, lane, + amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)); } } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp index d92cc674bf32..906bf9adf7c3 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp @@ -2763,17 +2763,12 @@ void VAddMaxI32Vop3::execute_impl(amdgpu::Wavefront &wf) { for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, [&]() { - uint32_t sum_bits = - static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))) + - static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane))); - int32_t sum = static_cast(sum_bits); - int32_t clamp = static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane))); - return static_cast(std::max(sum, clamp)); - }()); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_add_minmax( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)), inst_.clamp)); } } @@ -2797,17 +2792,12 @@ RJ_NOINLINE void VAddMaxI32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, [&]() { - uint32_t sum_bits = - static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))) + - static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane))); - int32_t sum = static_cast(sum_bits); - int32_t clamp = static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane))); - return static_cast(std::max(sum, clamp)); - }()); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_add_minmax( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)), inst_.clamp)); } dpp_write_mask_scope_.restore(); } @@ -2821,11 +2811,12 @@ void VAddMaxU32Vop3::execute_impl(amdgpu::Wavefront &wf) { for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - amdgpu::sdwa::write_lane( - *this, wf, vdst, lane, - std::max(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)) + - static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)), - static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)))); + amdgpu::sdwa::write_lane(*this, wf, vdst, lane, + amdgpu::vop3_integer_add_minmax( + amdgpu::RegisterAccess(wf).read_lane(src0, lane), + amdgpu::RegisterAccess(wf).read_lane(src1, lane), + amdgpu::RegisterAccess(wf).read_lane(src2, lane), + inst_.clamp)); } } @@ -2849,11 +2840,12 @@ RJ_NOINLINE void VAddMaxU32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - amdgpu::sdwa::write_lane( - *this, wf, vdst, lane, - std::max(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)) + - static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)), - static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)))); + amdgpu::sdwa::write_lane(*this, wf, vdst, lane, + amdgpu::vop3_integer_add_minmax( + amdgpu::RegisterAccess(wf).read_lane(src0, lane), + amdgpu::RegisterAccess(wf).read_lane(src1, lane), + amdgpu::RegisterAccess(wf).read_lane(src2, lane), + inst_.clamp)); } dpp_write_mask_scope_.restore(); } @@ -2867,17 +2859,12 @@ void VAddMinI32Vop3::execute_impl(amdgpu::Wavefront &wf) { for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, [&]() { - uint32_t sum_bits = - static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))) + - static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane))); - int32_t sum = static_cast(sum_bits); - int32_t clamp = static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane))); - return static_cast(std::min(sum, clamp)); - }()); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_add_minmax( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)), inst_.clamp)); } } @@ -2901,17 +2888,12 @@ RJ_NOINLINE void VAddMinI32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, [&]() { - uint32_t sum_bits = - static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))) + - static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane))); - int32_t sum = static_cast(sum_bits); - int32_t clamp = static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane))); - return static_cast(std::min(sum, clamp)); - }()); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_add_minmax( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)), inst_.clamp)); } dpp_write_mask_scope_.restore(); } @@ -2925,11 +2907,12 @@ void VAddMinU32Vop3::execute_impl(amdgpu::Wavefront &wf) { for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - amdgpu::sdwa::write_lane( - *this, wf, vdst, lane, - std::min(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)) + - static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)), - static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)))); + amdgpu::sdwa::write_lane(*this, wf, vdst, lane, + amdgpu::vop3_integer_add_minmax( + amdgpu::RegisterAccess(wf).read_lane(src0, lane), + amdgpu::RegisterAccess(wf).read_lane(src1, lane), + amdgpu::RegisterAccess(wf).read_lane(src2, lane), + inst_.clamp)); } } @@ -2953,11 +2936,12 @@ RJ_NOINLINE void VAddMinU32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - amdgpu::sdwa::write_lane( - *this, wf, vdst, lane, - std::min(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)) + - static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)), - static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)))); + amdgpu::sdwa::write_lane(*this, wf, vdst, lane, + amdgpu::vop3_integer_add_minmax( + amdgpu::RegisterAccess(wf).read_lane(src0, lane), + amdgpu::RegisterAccess(wf).read_lane(src1, lane), + amdgpu::RegisterAccess(wf).read_lane(src2, lane), + inst_.clamp)); } dpp_write_mask_scope_.restore(); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vop3_exec.cpp index 6cd9f37de85a..961623400c62 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vop3_exec.cpp @@ -645,7 +645,9 @@ void VQsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -669,7 +671,9 @@ void VMqsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -711,7 +715,9 @@ void VMqsadU32U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } dst_word.encoding_value_ = vdst.encoding_value_ + static_cast(window); - amdgpu::sdwa::write_lane(*this, wf, dst_word, lane, accum[window] + sum); + amdgpu::sdwa::write_lane( + *this, wf, dst_word, lane, + amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)); } } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vop3_exec.cpp index 5df3ac53f9cf..ea6b2fbb05b3 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vop3_exec.cpp @@ -687,7 +687,9 @@ void VQsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -711,7 +713,9 @@ void VMqsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -753,7 +757,9 @@ void VMqsadU32U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } dst_word.encoding_value_ = vdst.encoding_value_ + static_cast(window); - amdgpu::sdwa::write_lane(*this, wf, dst_word, lane, accum[window] + sum); + amdgpu::sdwa::write_lane( + *this, wf, dst_word, lane, + amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)); } } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop3_exec.cpp index 25afe8e7d161..5efbefe76445 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop3_exec.cpp @@ -6612,7 +6612,9 @@ void VQsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -6636,7 +6638,9 @@ void VMqsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -6674,7 +6678,9 @@ void VMqsadU32U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } dst_word.encoding_value_ = vdst.encoding_value_ + static_cast(window); - amdgpu::sdwa::write_lane(*this, wf, dst_word, lane, accum[window] + sum); + amdgpu::sdwa::write_lane( + *this, wf, dst_word, lane, + amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)); } } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop3_exec.cpp index ae86828965e9..45b3ab8c8c18 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop3_exec.cpp @@ -6612,7 +6612,9 @@ void VQsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -6636,7 +6638,9 @@ void VMqsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -6674,7 +6678,9 @@ void VMqsadU32U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } dst_word.encoding_value_ = vdst.encoding_value_ + static_cast(window); - amdgpu::sdwa::write_lane(*this, wf, dst_word, lane, accum[window] + sum); + amdgpu::sdwa::write_lane( + *this, wf, dst_word, lane, + amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)); } } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3_exec.cpp index d694022f3388..ccef57f1c9b0 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3_exec.cpp @@ -7649,7 +7649,9 @@ void VQsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -7673,7 +7675,9 @@ void VMqsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -7711,7 +7715,9 @@ void VMqsadU32U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } dst_word.encoding_value_ = vdst.encoding_value_ + static_cast(window); - amdgpu::sdwa::write_lane(*this, wf, dst_word, lane, accum[window] + sum); + amdgpu::sdwa::write_lane( + *this, wf, dst_word, lane, + amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)); } } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h index f4c799b59011..013c2ae82be1 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h @@ -119,6 +119,23 @@ inline std::make_unsigned_t vop3_integer_add(std::make_unsigned_t lhs, } } +/// @brief Add with optional saturation, then select the maximum or minimum operand. +template +inline std::make_unsigned_t vop3_integer_add_minmax(std::make_unsigned_t lhs, + std::make_unsigned_t rhs, + std::make_unsigned_t bound, bool clamp) { + static_assert(std::is_integral_v && sizeof(T) == 4); + using U = std::make_unsigned_t; + const U sum_bits = vop3_integer_add(lhs, rhs, clamp); + if constexpr (std::is_signed_v) { + const T sum = std::bit_cast(sum_bits); + const T typed_bound = std::bit_cast(bound); + return static_cast(SelectMax ? std::max(sum, typed_bound) : std::min(sum, typed_bound)); + } else { + return SelectMax ? std::max(sum_bits, bound) : std::min(sum_bits, bound); + } +} + /// @brief Execute an integer multiply-add with an exact intermediate. template inline std::make_unsigned_t vop3_integer_mad(uint32_t lhs, uint32_t rhs, diff --git a/emulation/rocjitsu/tests/cdna5_execution_test.cpp b/emulation/rocjitsu/tests/cdna5_execution_test.cpp index f31ce3b04219..2a9e8bf86f7e 100644 --- a/emulation/rocjitsu/tests/cdna5_execution_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_execution_test.cpp @@ -263,7 +263,15 @@ TEST(Gfx1250ExecutionTest, Vop3IntegerMultiplyAccumulateClampSaturatesExactResul TestCase{cdna5::kVMadI32I16Vop3, static_cast(INT16_MIN), static_cast(INT16_MAX), static_cast(INT32_MIN), static_cast(INT32_MIN)}, + TestCase{cdna5::kVAddMaxI32Vop3, static_cast(INT32_MAX), 1u, + static_cast(INT32_MIN), static_cast(INT32_MAX)}, + TestCase{cdna5::kVAddMaxU32Vop3, UINT32_MAX, 1u, 0u, UINT32_MAX}, + TestCase{cdna5::kVAddMinI32Vop3, static_cast(INT32_MIN), static_cast(-1), + static_cast(INT32_MAX), static_cast(INT32_MIN)}, + TestCase{cdna5::kVAddMinU32Vop3, UINT32_MAX, 1u, UINT32_MAX, UINT32_MAX}, TestCase{cdna5::kVSadHiU8Vop3, UINT32_MAX, 0u, 0xfc040000u, UINT32_MAX}, + TestCase{cdna5::kVSadU8Vop3, 0x00010203u, 0x10002030u, UINT32_MAX, UINT32_MAX}, + TestCase{cdna5::kVSadU16Vop3, 0x00010003u, 0x10002000u, UINT32_MAX, UINT32_MAX}, TestCase{cdna5::kVSadU32Vop3, 0u, UINT32_MAX, 1u, UINT32_MAX}, TestCase{cdna5::kVMsadU8Vop3, 0xffffffffu, 0x01010101u, UINT32_MAX, UINT32_MAX}, }; @@ -3066,6 +3074,39 @@ TEST(Gfx1250ExecutionTest, QsadAndMqsadUseFourSlidingWindowsAndMaskedReferenceBy EXPECT_EQ(read(6), 202u); EXPECT_EQ(read(7), 304u); EXPECT_EQ(read(8), 406u); + + // CLAMP saturates each packed or full-width accumulation independently. + write(0, 0u); + write(1, 0u); + write(2, 0xffff'ffffu); + write(4, 0xffff'ffffu); + write(5, 0xffff'ffffu); + const auto clamped_qsad_words = cdna5::build_vop3( + cdna5::kVQsadPkU16U8Vop3, {.vdst = 8, .clamp = 1, .src0 = 256, .src1 = 258, .src2 = 260}); + cdna5::VQsadPkU16U8Vop3 clamped_qsad(clamped_qsad_words.data()); + clamped_qsad.execute_impl(*wf); + EXPECT_EQ(read(8), UINT32_MAX); + EXPECT_EQ(read(9), UINT32_MAX); + + const auto clamped_mqsad_pk_words = cdna5::build_vop3( + cdna5::kVMqsadPkU16U8Vop3, {.vdst = 8, .clamp = 1, .src0 = 256, .src1 = 258, .src2 = 260}); + cdna5::VMqsadPkU16U8Vop3 clamped_mqsad_pk(clamped_mqsad_pk_words.data()); + clamped_mqsad_pk.execute_impl(*wf); + EXPECT_EQ(read(8), UINT32_MAX); + EXPECT_EQ(read(9), UINT32_MAX); + + write(4, UINT32_MAX); + write(5, UINT32_MAX); + write(6, UINT32_MAX); + write(7, UINT32_MAX); + const auto clamped_mqsad_words = cdna5::build_vop3( + cdna5::kVMqsadU32U8Vop3, {.vdst = 12, .clamp = 1, .src0 = 256, .src1 = 258, .src2 = 260}); + cdna5::VMqsadU32U8Vop3 clamped_mqsad(clamped_mqsad_words.data()); + clamped_mqsad.execute_impl(*wf); + EXPECT_EQ(read(12), UINT32_MAX); + EXPECT_EQ(read(13), UINT32_MAX); + EXPECT_EQ(read(14), UINT32_MAX); + EXPECT_EQ(read(15), UINT32_MAX); } TEST(Gfx1250ExecutionTest, MullitAndTrigPreopPreserveLegacyAndRangeReductionRules) { From ab4b68906bac45d043eeafe2d13e6eb3b1a6279a Mon Sep 17 00:00:00 2001 From: Sambhav Jain Date: Wed, 2 Sep 2026 20:52:38 +0000 Subject: [PATCH 09/16] [rocjitsu] Cover unclamped integer minmax addition Exercise the legacy modulo-before-selection behavior alongside the new clamped boundary cases. This guards the shared ADD_MIN/MAX helper from saturating unconditionally. Signed-off-by: Sambhav Jain --- emulation/rocjitsu/tests/cdna5_execution_test.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/emulation/rocjitsu/tests/cdna5_execution_test.cpp b/emulation/rocjitsu/tests/cdna5_execution_test.cpp index 2a9e8bf86f7e..c238b86c5fd7 100644 --- a/emulation/rocjitsu/tests/cdna5_execution_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_execution_test.cpp @@ -265,10 +265,16 @@ TEST(Gfx1250ExecutionTest, Vop3IntegerMultiplyAccumulateClampSaturatesExactResul static_cast(INT32_MIN)}, TestCase{cdna5::kVAddMaxI32Vop3, static_cast(INT32_MAX), 1u, static_cast(INT32_MIN), static_cast(INT32_MAX)}, + TestCase{cdna5::kVAddMaxI32Vop3, static_cast(INT32_MAX), 1u, + static_cast(INT32_MIN), static_cast(INT32_MIN), false}, TestCase{cdna5::kVAddMaxU32Vop3, UINT32_MAX, 1u, 0u, UINT32_MAX}, + TestCase{cdna5::kVAddMaxU32Vop3, UINT32_MAX, 1u, 0u, 0u, false}, TestCase{cdna5::kVAddMinI32Vop3, static_cast(INT32_MIN), static_cast(-1), static_cast(INT32_MAX), static_cast(INT32_MIN)}, + TestCase{cdna5::kVAddMinI32Vop3, static_cast(INT32_MIN), static_cast(-1), + static_cast(INT32_MAX), static_cast(INT32_MAX), false}, TestCase{cdna5::kVAddMinU32Vop3, UINT32_MAX, 1u, UINT32_MAX, UINT32_MAX}, + TestCase{cdna5::kVAddMinU32Vop3, UINT32_MAX, 1u, UINT32_MAX, 0u, false}, TestCase{cdna5::kVSadHiU8Vop3, UINT32_MAX, 0u, 0xfc040000u, UINT32_MAX}, TestCase{cdna5::kVSadU8Vop3, 0x00010203u, 0x10002030u, UINT32_MAX, UINT32_MAX}, TestCase{cdna5::kVSadU16Vop3, 0x00010003u, 0x10002000u, UINT32_MAX, UINT32_MAX}, From affb38584e7621f104698dd7bbe79bad1dd0526a Mon Sep 17 00:00:00 2001 From: Sambhav Jain Date: Wed, 2 Sep 2026 22:58:43 +0000 Subject: [PATCH 10/16] [rocjitsu] Address remaining integer clamp semantics Signed-off-by: Sambhav Jain --- .../python/amdisa/codegen/execute/__init__.py | 7 +- .../python/amdisa/codegen/execute/packed.py | 21 ++ .../amdisa/codegen/execute/sema_lower.py | 3 +- .../amdisa/codegen/execute/vector_special.py | 33 ++- .../rocjitsu/lib/python/amdisa/isa_profile.py | 12 +- .../tests/test_generator_profile_gates.py | 47 ++++- .../amdisa/tests/test_packed_codegen.py | 26 +++ .../python/amdisa/tests/test_sema_derive.py | 4 +- .../generated/cdna5/vop3_exec_ternary.cpp | 56 +++-- .../amdgpu/generated/cdna5/vop3p_exec.cpp | 12 +- .../amdgpu/generated/rdna4/vop3p_exec.cpp | 12 +- .../amdgpu/generated/shared/execute_shared.h | 50 +++-- .../isa/arch/amdgpu/shared/simd_glue.h | 15 +- .../rocjitsu/tests/cdna5_execution_test.cpp | 195 +++++++++++++++++- 14 files changed, 412 insertions(+), 81 deletions(-) diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py index ceb6461a10ef..a0c25518bde8 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py @@ -218,7 +218,11 @@ def _register_handlers() -> None: c.dst_ops, c.src_ops, c.is_vop3, c.has_abs ) DISPATCH['vector_mad_64_32'] = lambda c: gen_vector_mad_64_32( - c.dst_ops, c.src_ops, c.dtype, c.result_writer + c.dst_ops, + c.src_ops, + c.dtype, + c.result_writer, + c.is_vop3 and c.inst.name in c.profile.integer_clamp_dtypes, ) DISPATCH['vector_mad_32_16'] = lambda c: gen_vector_mad_32_16( c.dst_ops, c.src_ops, c.dtype, c.is_vop3 @@ -313,6 +317,7 @@ def _register_handlers() -> None: c.dtype, op_sel_hi_2_expr=c.op_sel_hi_2_expr, opsel_exprs=c.opsel_exprs, + integer_clamp=c.inst.name in c.profile.integer_clamp_dtypes, ) DISPATCH['pk_fmac_vop2'] = lambda c: ( gen_pk_fmac_vop3(c.dst_ops, c.src_ops) diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py index e17aa9d3f43d..82eb15802e4c 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py @@ -249,6 +249,7 @@ def gen_pk_ternary( dtype: str | None, op_sel_hi_2_expr: str = '', opsel_exprs: tuple[str, str] = ('', ''), + integer_clamp: bool = False, ) -> str: """Generate packed 16-bit ternary op (V_PK_FMA_F16, V_PK_MAD_I16, etc.).""" d, s0, s1, s2 = dst[0], src[0], src[1], src[2] @@ -432,6 +433,17 @@ def gen_pk_ternary( L.append( ' uint16_t rhi = static_cast(std::max(std::max(a_hi, b_hi), c_hi));' ) + elif op == 'mad' and integer_clamp: + L.append( + ' uint16_t rlo = amdgpu::vop3_integer_mad(' + 'static_cast(a_lo), static_cast(b_lo), ' + 'static_cast(c_lo), inst_.clamp);' + ) + L.append( + ' uint16_t rhi = amdgpu::vop3_integer_mad(' + 'static_cast(a_hi), static_cast(b_hi), ' + 'static_cast(c_hi), inst_.clamp);' + ) else: L.append( ' uint16_t rlo = static_cast(static_cast(a_lo) * b_lo + c_lo);' @@ -494,6 +506,15 @@ def gen_pk_ternary( elif op in ('max3', 'maximum3'): L.append(' uint16_t rlo = std::max(std::max(a_lo, b_lo), c_lo);') L.append(' uint16_t rhi = std::max(std::max(a_hi, b_hi), c_hi);') + elif op == 'mad' and integer_clamp: + L.append( + ' uint16_t rlo = amdgpu::vop3_integer_mad(' + 'a_lo, b_lo, c_lo, inst_.clamp);' + ) + L.append( + ' uint16_t rhi = amdgpu::vop3_integer_mad(' + 'a_hi, b_hi, c_hi, inst_.clamp);' + ) else: L.append( ' uint16_t rlo = static_cast(static_cast(a_lo) * b_lo + c_lo);' diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py index a6bb99ef6b2c..6146ab8752e9 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py @@ -1772,8 +1772,7 @@ def _lower_call(node: SemaNode, ctx: LoweringContext) -> str: if ctx.integer_saturation_dtype is not None and callee in add_minmax_calls: cpp_type, select_max = add_minmax_calls[callee] return ( - f'amdgpu::vop3_integer_add_minmax<{cpp_type}, {select_max}>(' - f'{args_str}, inst_.clamp)' + f'amdgpu::vop3_integer_add_minmax<{cpp_type}, {select_max}>(' f'{args_str})' ) saturating_calls = { diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py index 4906b8bfb68d..039126ba1264 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py @@ -710,6 +710,7 @@ def gen_vector_mad_64_32( src: list[str], dtype: str | None, result_writer: str | None = None, + integer_clamp: bool = False, ) -> str: """Generate V_MAD_U64_U32 / V_MAD_I64_I32 body. @@ -735,18 +736,24 @@ def gen_vector_mad_64_32( f' int64_t s1 = static_cast(amdgpu::RegisterAccess(wf).read_lane({src[1]}, lane));' ) L.append( - f' uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64({src[2]}, lane);' - ) - L.append( - ' uint64_t product = static_cast(s0) * static_cast(s1);' + f' int64_t s2 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64({src[2]}, lane));' ) - L.append(' uint64_t result = product + s2;') + L.append(' __int128 wide = static_cast<__int128>(s0) * s1 + s2;') + L.append(' uint64_t result = static_cast(wide);') if writes_carry: - L.append(' constexpr uint64_t sign_bit = 1ULL << 63;') L.append( - ' if (((~(product ^ s2) & (product ^ result)) & sign_bit) != 0)' + ' if (wide < std::numeric_limits::min() || wide > std::numeric_limits::max())' ) L.append(' carry |= 1ULL << lane;') + if integer_clamp: + L.append(' if (inst_.clamp) {') + L.append( + ' if (wide < std::numeric_limits::min()) result = std::bit_cast(std::numeric_limits::min());' + ) + L.append( + ' else if (wide > std::numeric_limits::max()) result = static_cast(std::numeric_limits::max());' + ) + L.append(' }') L.append( f' amdgpu::RegisterAccess(wf).write_lane64({dst[0]}, lane, result);' ) @@ -760,11 +767,17 @@ def gen_vector_mad_64_32( L.append( f' uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64({src[2]}, lane);' ) - L.append(' uint64_t product = s0 * s1;') - L.append(' uint64_t result = product + s2;') + L.append( + ' unsigned __int128 wide = static_cast(s0) * s1 + s2;' + ) + L.append(' uint64_t result = static_cast(wide);') if writes_carry: - L.append(' if (result < product)') + L.append(' if (wide > std::numeric_limits::max())') L.append(' carry |= 1ULL << lane;') + if integer_clamp: + L.append( + ' if (inst_.clamp && wide > std::numeric_limits::max()) result = std::numeric_limits::max();' + ) L.append( f' amdgpu::RegisterAccess(wf).write_lane64({dst[0]}, lane, result);' ) diff --git a/emulation/rocjitsu/lib/python/amdisa/isa_profile.py b/emulation/rocjitsu/lib/python/amdisa/isa_profile.py index e827ecd2fe5d..71a89447e17b 100644 --- a/emulation/rocjitsu/lib/python/amdisa/isa_profile.py +++ b/emulation/rocjitsu/lib/python/amdisa/isa_profile.py @@ -582,11 +582,13 @@ def vop3p_source_modifier_omissions(self) -> frozenset[str]: @property def integer_clamp_dtypes(self) -> dict[str, str]: - """Integer VOP3 instructions whose CLAMP modifier saturates the result. + """Integer instructions that require saturation-aware lowering. CLAMP is present in the shared VOP3 encoding, but it is not legal for every opcode in that encoding. Keep the instruction-level policy here - instead of inferring support from the encoding field alone. + instead of inferring support from the encoding field alone. ADD_MIN/MAX + always saturate their internal addition; the remaining instructions use + this policy to apply their encoded CLAMP modifier. """ return { 'V_ADDC_CO_U32': 'u32', @@ -608,6 +610,10 @@ def integer_clamp_dtypes(self) -> dict[str, str]: 'V_MAD_I16': 'i16', 'V_MAD_I32_I16': 'i32', 'V_MAD_I32_I24': 'i32', + 'V_MAD_CO_I64_I32': 'i64', + 'V_MAD_CO_U64_U32': 'u64', + 'V_MAD_NC_I64_I32': 'i64', + 'V_MAD_NC_U64_U32': 'u64', 'V_MAD_U16': 'u16', 'V_MAD_U32_U16': 'u32', 'V_MAD_U32_U24': 'u32', @@ -616,6 +622,8 @@ def integer_clamp_dtypes(self) -> dict[str, str]: 'V_MSAD_U8': 'u32', 'V_MUL_I32_I24': 'i32', 'V_MUL_U32_U24': 'u32', + 'V_PK_MAD_I16': 'i16', + 'V_PK_MAD_U16': 'u16', 'V_SAD_HI_U8': 'u32', 'V_SAD_U8': 'u32', 'V_SAD_U16': 'u32', diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py index 3a62d0e1b263..d4ec1c80934f 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py @@ -1754,8 +1754,8 @@ def test_vop3_mad_u64_u32_writes_explicit_sdst_carry(): body = gen_vector_mad_64_32(['vdst', 'sdst'], ['src0', 'src1', 'src2'], 'u64') assert 'uint64_t carry = 0;' in body - assert 'uint64_t product = s0 * s1;' in body - assert 'if (result < product)' in body + assert 'unsigned __int128 wide' in body + assert 'if (wide > std::numeric_limits::max())' in body assert 'carry |= 1ULL << lane;' in body assert 'amdgpu::write_wave_mask_scalar(sdst, wf, carry);' in body assert 'wf.set_vcc' not in body @@ -1770,6 +1770,29 @@ def test_vop3_mad_u64_u32_writes_explicit_sdst_carry(): assert 'write_wave_mask_scalar' not in callback_body +def test_vop3_mad_64_32_clamps_exact_result_without_changing_carry(): + unsigned = gen_vector_mad_64_32( + ['vdst', 'sdst'], + ['src0', 'src1', 'src2'], + 'u64', + integer_clamp=True, + ) + signed = gen_vector_mad_64_32( + ['vdst', 'sdst'], + ['src0', 'src1', 'src2'], + 'i64', + integer_clamp=True, + ) + + assert 'unsigned __int128 wide' in unsigned + assert 'inst_.clamp && wide > std::numeric_limits::max()' in unsigned + assert unsigned.index('if (wide >') < unsigned.index('if (inst_.clamp') + assert '__int128 wide' in signed + assert 'wide < std::numeric_limits::min()' in signed + assert 'if (inst_.clamp)' in signed + assert signed.index('carry |= 1ULL << lane') < signed.index('if (inst_.clamp)') + + def test_vector_cmp_class_writes_explicit_sdst_mask(): body = gen_vector_cmp_class( ['sdst'], ['src0', 'src1'], 'f32', is_cmpx=False, is_vop3=True @@ -4077,7 +4100,7 @@ def test_gfx1250_generated_vop3_add_f16_applies_dpp( assert 'src0.clear_delegate();' not in body -def test_gfx1250_generator_wires_integer_clamp_only_from_encoded_vop3_field(): +def test_gfx1250_generator_wires_instruction_specific_integer_saturation(): isa_xml = _mrisa_dir() / 'amdgpu_isa_cdna5.xml' parser = Parser(str(isa_xml), Cdna5Profile()) spec = parser.parse() @@ -4123,8 +4146,26 @@ def generated_body(name: str, enc_name: str, *, result_writer=None) -> str: for name, helper in expected_helpers.items(): body = generated_body(name, 'ENC_VOP3') assert helper in body + assert 'inst_.clamp' not in body + + for name, enc_name, result_writer in ( + ('V_MAD_NC_U64_U32', 'ENC_VOP3', None), + ('V_MAD_NC_I64_I32', 'ENC_VOP3', None), + ('V_MAD_CO_U64_U32', 'VOP3_SDST_ENC', 'commit_result'), + ('V_MAD_CO_I64_I32', 'VOP3_SDST_ENC', 'commit_result'), + ): + body = generated_body(name, enc_name, result_writer=result_writer) + assert '__int128 wide' in body assert 'inst_.clamp' in body + for name, helper in ( + ('V_PK_MAD_I16', 'vop3_integer_mad'), + ('V_PK_MAD_U16', 'vop3_integer_mad'), + ): + body = generated_body(name, 'ENC_VOP3P') + assert body.count(helper) == 2 + assert body.count('inst_.clamp') == 2 + for name in ('V_QSAD_PK_U16_U8', 'V_MQSAD_PK_U16_U8', 'V_MQSAD_U32_U8'): body = generated_body(name, 'ENC_VOP3') assert 'inst_.clamp' in body diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py index 97b094478b18..8ffe441572d2 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py @@ -121,6 +121,32 @@ def test_pk_add_minmax_saturates_add_before_selecting_third_operand(): assert 'std::min(sum_lo, c_lo)' in unsigned +def test_pk_mad_integer_clamp_saturates_both_selected_halves(): + signed = gen_pk_ternary( + ['vdst'], + ['src0', 'src1', 'src2'], + 'mad', + 'i16', + op_sel_hi_2_expr='inst_.op_sel_hi_2', + opsel_exprs=('inst_.op_sel', 'inst_.op_sel_hi'), + integer_clamp=True, + ) + unsigned = gen_pk_ternary( + ['vdst'], + ['src0', 'src1', 'src2'], + 'mad', + 'u16', + op_sel_hi_2_expr='inst_.op_sel_hi_2', + opsel_exprs=('inst_.op_sel', 'inst_.op_sel_hi'), + integer_clamp=True, + ) + + assert signed.count('vop3_integer_mad') == 2 + assert signed.count('inst_.clamp') == 2 + assert unsigned.count('vop3_integer_mad') == 2 + assert unsigned.count('inst_.clamp') == 2 + + def test_dot8_iu4_uses_operand_signedness_modifiers(): cpp = gen_dot8(['vdst'], ['src0', 'src1', 'src2'], 'dot8_i32_iu4') diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py index cf931d5fbb49..fc616ac1ebd2 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py @@ -1705,7 +1705,7 @@ def test_lowers_to_std_fma(self): cpp = lower_sema_block(block) assert 'std::fma(' in cpp - def test_add_minmax_i32_u32_can_saturate_add_before_selection(self): + def test_add_minmax_i32_u32_intrinsically_saturates_add_before_selection(self): cases = [ ('V_ADD_MAX_I32', 'add_max', 'i32', 'std::max'), ('V_ADD_MAX_U32', 'add_max', 'u32', 'std::max'), @@ -1730,7 +1730,7 @@ def test_add_minmax_i32_u32_can_saturate_add_before_selection(self): select_max = 'true' if clamp_fn == 'std::max' else 'false' assert f'vop3_integer_add_minmax<' in cpp assert f', {select_max}>' in cpp - assert 'inst_.clamp' in cpp + assert 'inst_.clamp' not in cpp def test_ashr_pk_i8_u8_i32_packs_shifted_saturated_bytes(self): cases = [ diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp index 906bf9adf7c3..be61e9361b45 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp @@ -2768,7 +2768,7 @@ void VAddMaxI32Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::vop3_integer_add_minmax( static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)), static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)), - static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)), inst_.clamp)); + static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)))); } } @@ -2797,7 +2797,7 @@ RJ_NOINLINE void VAddMaxI32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::vop3_integer_add_minmax( static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)), static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)), - static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)), inst_.clamp)); + static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)))); } dpp_write_mask_scope_.restore(); } @@ -2815,8 +2815,7 @@ void VAddMaxU32Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::vop3_integer_add_minmax( amdgpu::RegisterAccess(wf).read_lane(src0, lane), amdgpu::RegisterAccess(wf).read_lane(src1, lane), - amdgpu::RegisterAccess(wf).read_lane(src2, lane), - inst_.clamp)); + amdgpu::RegisterAccess(wf).read_lane(src2, lane))); } } @@ -2844,8 +2843,7 @@ RJ_NOINLINE void VAddMaxU32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::vop3_integer_add_minmax( amdgpu::RegisterAccess(wf).read_lane(src0, lane), amdgpu::RegisterAccess(wf).read_lane(src1, lane), - amdgpu::RegisterAccess(wf).read_lane(src2, lane), - inst_.clamp)); + amdgpu::RegisterAccess(wf).read_lane(src2, lane))); } dpp_write_mask_scope_.restore(); } @@ -2864,7 +2862,7 @@ void VAddMinI32Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::vop3_integer_add_minmax( static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)), static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)), - static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)), inst_.clamp)); + static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)))); } } @@ -2893,7 +2891,7 @@ RJ_NOINLINE void VAddMinI32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::vop3_integer_add_minmax( static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)), static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)), - static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)), inst_.clamp)); + static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)))); } dpp_write_mask_scope_.restore(); } @@ -2911,8 +2909,7 @@ void VAddMinU32Vop3::execute_impl(amdgpu::Wavefront &wf) { amdgpu::vop3_integer_add_minmax( amdgpu::RegisterAccess(wf).read_lane(src0, lane), amdgpu::RegisterAccess(wf).read_lane(src1, lane), - amdgpu::RegisterAccess(wf).read_lane(src2, lane), - inst_.clamp)); + amdgpu::RegisterAccess(wf).read_lane(src2, lane))); } } @@ -2940,8 +2937,7 @@ RJ_NOINLINE void VAddMinU32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { amdgpu::vop3_integer_add_minmax( amdgpu::RegisterAccess(wf).read_lane(src0, lane), amdgpu::RegisterAccess(wf).read_lane(src1, lane), - amdgpu::RegisterAccess(wf).read_lane(src2, lane), - inst_.clamp)); + amdgpu::RegisterAccess(wf).read_lane(src2, lane))); } dpp_write_mask_scope_.restore(); } @@ -3976,8 +3972,10 @@ void VMadNcU64U32Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(src2, lane); - uint64_t product = s0 * s1; - uint64_t result = product + s2; + unsigned __int128 wide = static_cast(s0) * s1 + s2; + uint64_t result = static_cast(wide); + if (inst_.clamp && wide > std::numeric_limits::max()) + result = std::numeric_limits::max(); amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, result); } } @@ -4003,8 +4001,10 @@ RJ_NOINLINE void VMadNcU64U32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(src2, lane); - uint64_t product = s0 * s1; - uint64_t result = product + s2; + unsigned __int128 wide = static_cast(s0) * s1 + s2; + uint64_t result = static_cast(wide); + if (inst_.clamp && wide > std::numeric_limits::max()) + result = std::numeric_limits::max(); amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, result); } dpp_write_mask_scope_.restore(); @@ -4021,9 +4021,15 @@ void VMadNcI64I32Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; int64_t s0 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); int64_t s1 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(src2, lane); - uint64_t product = static_cast(s0) * static_cast(s1); - uint64_t result = product + s2; + int64_t s2 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src2, lane)); + __int128 wide = static_cast<__int128>(s0) * s1 + s2; + uint64_t result = static_cast(wide); + if (inst_.clamp) { + if (wide < std::numeric_limits::min()) + result = std::bit_cast(std::numeric_limits::min()); + else if (wide > std::numeric_limits::max()) + result = static_cast(std::numeric_limits::max()); + } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, result); } } @@ -4048,9 +4054,15 @@ RJ_NOINLINE void VMadNcI64I32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) continue; int64_t s0 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); int64_t s1 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(src2, lane); - uint64_t product = static_cast(s0) * static_cast(s1); - uint64_t result = product + s2; + int64_t s2 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src2, lane)); + __int128 wide = static_cast<__int128>(s0) * s1 + s2; + uint64_t result = static_cast(wide); + if (inst_.clamp) { + if (wide < std::numeric_limits::min()) + result = std::bit_cast(std::numeric_limits::min()); + else if (wide > std::numeric_limits::max()) + result = static_cast(std::numeric_limits::max()); + } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, result); } dpp_write_mask_scope_.restore(); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp index 3bca13cc720b..e62628f702a7 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp @@ -159,8 +159,12 @@ void VPkMadI16Vop3p::execute_impl(amdgpu::Wavefront &wf) { int16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); int16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); int16_t c_hi = static_cast(sel2_hi ? (raw2 >> 16) : raw2); - uint16_t rlo = static_cast(static_cast(a_lo) * b_lo + c_lo); - uint16_t rhi = static_cast(static_cast(a_hi) * b_hi + c_hi); + uint16_t rlo = amdgpu::vop3_integer_mad(static_cast(a_lo), + static_cast(b_lo), + static_cast(c_lo), inst_.clamp); + uint16_t rhi = amdgpu::vop3_integer_mad(static_cast(a_hi), + static_cast(b_hi), + static_cast(c_hi), inst_.clamp); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -366,8 +370,8 @@ void VPkMadU16Vop3p::execute_impl(amdgpu::Wavefront &wf) { uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); uint16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); uint16_t c_hi = static_cast(sel2_hi ? (raw2 >> 16) : raw2); - uint16_t rlo = static_cast(static_cast(a_lo) * b_lo + c_lo); - uint16_t rhi = static_cast(static_cast(a_hi) * b_hi + c_hi); + uint16_t rlo = amdgpu::vop3_integer_mad(a_lo, b_lo, c_lo, inst_.clamp); + uint16_t rhi = amdgpu::vop3_integer_mad(a_hi, b_hi, c_hi, inst_.clamp); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3p_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3p_exec.cpp index 470468c7177d..66b169592f8e 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3p_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3p_exec.cpp @@ -44,8 +44,12 @@ void VPkMadI16Vop3p::execute_impl(amdgpu::Wavefront &wf) { int16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); int16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); int16_t c_hi = static_cast(sel2_hi ? (raw2 >> 16) : raw2); - uint16_t rlo = static_cast(static_cast(a_lo) * b_lo + c_lo); - uint16_t rhi = static_cast(static_cast(a_hi) * b_hi + c_hi); + uint16_t rlo = amdgpu::vop3_integer_mad(static_cast(a_lo), + static_cast(b_lo), + static_cast(c_lo), inst_.clamp); + uint16_t rhi = amdgpu::vop3_integer_mad(static_cast(a_hi), + static_cast(b_hi), + static_cast(c_hi), inst_.clamp); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -251,8 +255,8 @@ void VPkMadU16Vop3p::execute_impl(amdgpu::Wavefront &wf) { uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); uint16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); uint16_t c_hi = static_cast(sel2_hi ? (raw2 >> 16) : raw2); - uint16_t rlo = static_cast(static_cast(a_lo) * b_lo + c_lo); - uint16_t rhi = static_cast(static_cast(a_hi) * b_hi + c_hi); + uint16_t rlo = amdgpu::vop3_integer_mad(a_lo, b_lo, c_lo, inst_.clamp); + uint16_t rhi = amdgpu::vop3_integer_mad(a_hi, b_hi, c_hi, inst_.clamp); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h index 6f813e218fe8..8045b615353e 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h @@ -13200,12 +13200,17 @@ inline void execute_v_mad_co_i64_i32_vop3([[maybe_unused]] Inst &inst, continue; int64_t s0 = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); int64_t s1 = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(inst.src2, lane); - uint64_t product = static_cast(s0) * static_cast(s1); - uint64_t result = product + s2; - constexpr uint64_t sign_bit = 1ULL << 63; - if (((~(product ^ s2) & (product ^ result)) & sign_bit) != 0) + int64_t s2 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src2, lane)); + __int128 wide = static_cast<__int128>(s0) * s1 + s2; + uint64_t result = static_cast(wide); + if (wide < std::numeric_limits::min() || wide > std::numeric_limits::max()) carry |= 1ULL << lane; + if (inst.inst_.clamp) { + if (wide < std::numeric_limits::min()) + result = std::bit_cast(std::numeric_limits::min()); + else if (wide > std::numeric_limits::max()) + result = static_cast(std::numeric_limits::max()); + } sdwa::write_lane64(inst, wf, inst.vdst, lane, result); } commit_result(carry); @@ -13223,10 +13228,12 @@ inline void execute_v_mad_co_u64_u32_vop3([[maybe_unused]] Inst &inst, uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane); uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane); uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(inst.src2, lane); - uint64_t product = s0 * s1; - uint64_t result = product + s2; - if (result < product) + unsigned __int128 wide = static_cast(s0) * s1 + s2; + uint64_t result = static_cast(wide); + if (wide > std::numeric_limits::max()) carry |= 1ULL << lane; + if (inst.inst_.clamp && wide > std::numeric_limits::max()) + result = std::numeric_limits::max(); sdwa::write_lane64(inst, wf, inst.vdst, lane, result); } commit_result(carry); @@ -13370,11 +13377,10 @@ inline void execute_v_mad_i64_i32_vop3([[maybe_unused]] Inst &inst, [[maybe_unus continue; int64_t s0 = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); int64_t s1 = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(inst.src2, lane); - uint64_t product = static_cast(s0) * static_cast(s1); - uint64_t result = product + s2; - constexpr uint64_t sign_bit = 1ULL << 63; - if (((~(product ^ s2) & (product ^ result)) & sign_bit) != 0) + int64_t s2 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src2, lane)); + __int128 wide = static_cast<__int128>(s0) * s1 + s2; + uint64_t result = static_cast(wide); + if (wide < std::numeric_limits::min() || wide > std::numeric_limits::max()) carry |= 1ULL << lane; sdwa::write_lane64(inst, wf, inst.vdst, lane, result); } @@ -13634,9 +13640,9 @@ inline void execute_v_mad_u64_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unus uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane); uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane); uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(inst.src2, lane); - uint64_t product = s0 * s1; - uint64_t result = product + s2; - if (result < product) + unsigned __int128 wide = static_cast(s0) * s1 + s2; + uint64_t result = static_cast(wide); + if (wide > std::numeric_limits::max()) carry |= 1ULL << lane; sdwa::write_lane64(inst, wf, inst.vdst, lane, result); } @@ -17482,8 +17488,12 @@ inline void execute_v_pk_mad_i16_vop3p([[maybe_unused]] Inst &inst, int16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); int16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); int16_t c_hi = static_cast(sel2_hi ? (raw2 >> 16) : raw2); - uint16_t rlo = static_cast(static_cast(a_lo) * b_lo + c_lo); - uint16_t rhi = static_cast(static_cast(a_hi) * b_hi + c_hi); + uint16_t rlo = amdgpu::vop3_integer_mad( + static_cast(a_lo), static_cast(b_lo), static_cast(c_lo), + inst.inst_.clamp); + uint16_t rhi = amdgpu::vop3_integer_mad( + static_cast(a_hi), static_cast(b_hi), static_cast(c_hi), + inst.inst_.clamp); sdwa::write_lane(inst, wf, inst.vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -17522,8 +17532,8 @@ inline void execute_v_pk_mad_u16_vop3p([[maybe_unused]] Inst &inst, uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); uint16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); uint16_t c_hi = static_cast(sel2_hi ? (raw2 >> 16) : raw2); - uint16_t rlo = static_cast(static_cast(a_lo) * b_lo + c_lo); - uint16_t rhi = static_cast(static_cast(a_hi) * b_hi + c_hi); + uint16_t rlo = amdgpu::vop3_integer_mad(a_lo, b_lo, c_lo, inst.inst_.clamp); + uint16_t rhi = amdgpu::vop3_integer_mad(a_hi, b_hi, c_hi, inst.inst_.clamp); sdwa::write_lane(inst, wf, inst.vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h index 013c2ae82be1..a1e141e02f82 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h @@ -119,14 +119,14 @@ inline std::make_unsigned_t vop3_integer_add(std::make_unsigned_t lhs, } } -/// @brief Add with optional saturation, then select the maximum or minimum operand. +/// @brief Add with intrinsic saturation, then select the maximum or minimum operand. template inline std::make_unsigned_t vop3_integer_add_minmax(std::make_unsigned_t lhs, std::make_unsigned_t rhs, - std::make_unsigned_t bound, bool clamp) { + std::make_unsigned_t bound) { static_assert(std::is_integral_v && sizeof(T) == 4); using U = std::make_unsigned_t; - const U sum_bits = vop3_integer_add(lhs, rhs, clamp); + const U sum_bits = vop3_integer_add(lhs, rhs, true); if constexpr (std::is_signed_v) { const T sum = std::bit_cast(sum_bits); const T typed_bound = std::bit_cast(bound); @@ -3762,16 +3762,17 @@ template /// third source's high-half selector op_sel_hi_2 == 1 (per the scalar body /// `sel2_hi = inst.inst_.op_sel_hi_2`, which is a single bit — value 1 picks /// the high half for the high-output computation). For pk_mad_i16/u16 the -/// scalar bodies also do not apply neg/neg_hi/clamp; the SIMD path passes -/// through. Functor receives three u32 packed lane vectors and returns the -/// per-half-masked packed result. +/// scalar bodies do not apply neg/neg_hi. Clamped integer MAD falls back to +/// scalar execution for exact per-half saturation. Functor receives three u32 +/// packed lane vectors and returns the per-half-masked packed result. template requires(util::has_stdx_simd) [[nodiscard]] inline bool try_execute_vop3p_pk_ternary_int_simd(Inst &inst, Wavefront &wf, Op op) { if (simd_force_scalar() || !sdwa::supports_direct_simd_store(inst) || !inst.src0.simd_capable() || !inst.src1.simd_capable() || !inst.src2.simd_capable() || !inst.vdst.simd_capable()) return false; - if (inst.inst_.op_sel != 0u || inst.inst_.op_sel_hi != 3u || inst.inst_.op_sel_hi_2 != 1u) + if (inst.inst_.clamp || inst.inst_.op_sel != 0u || inst.inst_.op_sel_hi != 3u || + inst.inst_.op_sel_hi_2 != 1u) return false; using T = uint32_t; constexpr std::size_t W = util::native_width_v; diff --git a/emulation/rocjitsu/tests/cdna5_execution_test.cpp b/emulation/rocjitsu/tests/cdna5_execution_test.cpp index c238b86c5fd7..21431d7928e8 100644 --- a/emulation/rocjitsu/tests/cdna5_execution_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_execution_test.cpp @@ -266,15 +266,15 @@ TEST(Gfx1250ExecutionTest, Vop3IntegerMultiplyAccumulateClampSaturatesExactResul TestCase{cdna5::kVAddMaxI32Vop3, static_cast(INT32_MAX), 1u, static_cast(INT32_MIN), static_cast(INT32_MAX)}, TestCase{cdna5::kVAddMaxI32Vop3, static_cast(INT32_MAX), 1u, - static_cast(INT32_MIN), static_cast(INT32_MIN), false}, + static_cast(INT32_MIN), static_cast(INT32_MAX), false}, TestCase{cdna5::kVAddMaxU32Vop3, UINT32_MAX, 1u, 0u, UINT32_MAX}, - TestCase{cdna5::kVAddMaxU32Vop3, UINT32_MAX, 1u, 0u, 0u, false}, + TestCase{cdna5::kVAddMaxU32Vop3, UINT32_MAX, 1u, 0u, UINT32_MAX, false}, TestCase{cdna5::kVAddMinI32Vop3, static_cast(INT32_MIN), static_cast(-1), static_cast(INT32_MAX), static_cast(INT32_MIN)}, TestCase{cdna5::kVAddMinI32Vop3, static_cast(INT32_MIN), static_cast(-1), - static_cast(INT32_MAX), static_cast(INT32_MAX), false}, + static_cast(INT32_MAX), static_cast(INT32_MIN), false}, TestCase{cdna5::kVAddMinU32Vop3, UINT32_MAX, 1u, UINT32_MAX, UINT32_MAX}, - TestCase{cdna5::kVAddMinU32Vop3, UINT32_MAX, 1u, UINT32_MAX, 0u, false}, + TestCase{cdna5::kVAddMinU32Vop3, UINT32_MAX, 1u, UINT32_MAX, UINT32_MAX, false}, TestCase{cdna5::kVSadHiU8Vop3, UINT32_MAX, 0u, 0xfc040000u, UINT32_MAX}, TestCase{cdna5::kVSadU8Vop3, 0x00010203u, 0x10002030u, UINT32_MAX, UINT32_MAX}, TestCase{cdna5::kVSadU16Vop3, 0x00010003u, 0x10002000u, UINT32_MAX, UINT32_MAX}, @@ -371,6 +371,130 @@ TEST(Gfx1250ExecutionTest, Vop3U64ClampSaturatesNormalAndDppPaths) { } } +TEST(Gfx1250ExecutionTest, Vop3Mad64ClampSaturatesAndPreservesCarryNormalAndDpp) { + struct TestCase { + bool is_signed; + bool writes_carry; + uint32_t src0; + uint32_t src1; + uint64_t src2; + uint64_t expected; + }; + constexpr std::array kCases{ + TestCase{false, false, UINT32_MAX, UINT32_MAX, UINT64_MAX, UINT64_MAX}, + TestCase{false, true, UINT32_MAX, UINT32_MAX, UINT64_MAX, UINT64_MAX}, + TestCase{true, false, static_cast(INT32_MAX), static_cast(INT32_MAX), + static_cast(INT64_MAX), static_cast(INT64_MAX)}, + TestCase{true, true, static_cast(INT32_MIN), static_cast(INT32_MAX), + static_cast(INT64_MIN), static_cast(INT64_MIN)}, + }; + + for (bool dpp : {false, true}) { + for (const TestCase &test : kCases) { + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + const uint32_t base = wf->vgpr_alloc().base; + cu->write_vgpr(base, 0, test.src0); + cu->write_vgpr(base + 1, 0, test.src1); + cu->write_vgpr(base + 2, 0, static_cast(test.src2)); + cu->write_vgpr(base + 3, 0, static_cast(test.src2 >> 32)); + cu->write_vgpr(base + 4, 0, 0u); + cu->write_vgpr(base + 5, 0, 0u); + write_wave_sgpr(*cu, *wf, 2, 0u); + + if (!dpp) { + std::array words; + if (test.writes_carry) { + const uint16_t opcode = + test.is_signed ? cdna5::kVMadCoI64I32Vop3SdstEnc : cdna5::kVMadCoU64U32Vop3SdstEnc; + words = cdna5::build_vop3_sdst_enc( + opcode, {.vdst = 4, .sdst = 2, .clamp = 1, .src0 = 256, .src1 = 257, .src2 = 258}); + } else { + const uint16_t opcode = + test.is_signed ? cdna5::kVMadNcI64I32Vop3 : cdna5::kVMadNcU64U32Vop3; + words = cdna5::build_vop3(opcode, + {.vdst = 4, .clamp = 1, .src0 = 256, .src1 = 257, .src2 = 258}); + } + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA5); + ASSERT_NE(decoder, nullptr); + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + inst->execute(*inst, wf); + } else if (test.writes_carry) { + cdna5::Vop3SdstEncVopDpp16MachineInst raw{}; + raw.vdst = 4; + raw.sdst = 2; + raw.clamp = 1; + raw.src0 = amdgpu::SRC_DPP; + raw.src1 = 257; + raw.src2 = 258; + raw.vsrc0 = 0; + raw.dpp_ctrl = amdgpu::dpp::ROW_SELECT_BASE; + raw.bound_ctrl = 1; + raw.bank_mask = 0xF; + raw.row_mask = 0xF; + if (test.is_signed) { + cdna5::VMadCoI64I32Vop3SdstEnc inst(reinterpret_cast(&raw)); + inst.execute_impl(*wf); + } else { + cdna5::VMadCoU64U32Vop3SdstEnc inst(reinterpret_cast(&raw)); + inst.execute_impl(*wf); + } + } else { + cdna5::Vop3VopDpp16MachineInst raw{}; + raw.vdst = 4; + raw.clamp = 1; + raw.src0 = amdgpu::SRC_DPP; + raw.src1 = 257; + raw.src2 = 258; + raw.vsrc0 = 0; + raw.dpp_ctrl = amdgpu::dpp::ROW_SELECT_BASE; + raw.bound_ctrl = 1; + raw.bank_mask = 0xF; + raw.row_mask = 0xF; + if (test.is_signed) { + cdna5::VMadNcI64I32Vop3 inst(reinterpret_cast(&raw)); + inst.execute_impl(*wf); + } else { + cdna5::VMadNcU64U32Vop3 inst(reinterpret_cast(&raw)); + inst.execute_impl(*wf); + } + } + + const uint64_t result = static_cast(cu->read_vgpr(base + 4, 0)) | + (static_cast(cu->read_vgpr(base + 5, 0)) << 32); + EXPECT_EQ(result, test.expected) + << "signed " << test.is_signed << " co " << test.writes_carry << " dpp " << dpp; + if (test.writes_carry) + EXPECT_EQ(read_wave_sgpr(*cu, *wf, 2) & 1u, 1u); + } + } + + // CLAMP=0 keeps the modulo destination while CO still reports overflow. + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + const uint32_t base = wf->vgpr_alloc().base; + cu->write_vgpr(base, 0, UINT32_MAX); + cu->write_vgpr(base + 1, 0, UINT32_MAX); + cu->write_vgpr(base + 2, 0, UINT32_MAX); + cu->write_vgpr(base + 3, 0, UINT32_MAX); + const auto words = + cdna5::build_vop3_sdst_enc(cdna5::kVMadCoU64U32Vop3SdstEnc, + {.vdst = 4, .sdst = 2, .src0 = 256, .src1 = 257, .src2 = 258}); + cdna5::VMadCoU64U32Vop3SdstEnc inst(words.data()); + inst.execute_impl(*wf); + const uint64_t result = static_cast(cu->read_vgpr(base + 4, 0)) | + (static_cast(cu->read_vgpr(base + 5, 0)) << 32); + EXPECT_EQ(result, 0xfffffffe00000000ULL); + EXPECT_EQ(read_wave_sgpr(*cu, *wf, 2) & 1u, 1u); +} + TEST(Gfx1250ExecutionTest, Vop3PackedIntegerClampSaturatesSelectedHalves) { ForceScalarGuard guard; struct TestCase { @@ -409,6 +533,69 @@ TEST(Gfx1250ExecutionTest, Vop3PackedIntegerClampSaturatesSelectedHalves) { } } +TEST(Gfx1250ExecutionTest, Vop3PackedMadClampSaturatesSelectedHalves) { + ForceScalarGuard guard; + struct TestCase { + uint16_t opcode; + uint32_t src0; + uint32_t src1; + uint32_t src2; + uint32_t expected; + uint8_t opsel; + uint8_t opsel_hi; + uint8_t opsel_hi_2; + bool clamp = true; + }; + constexpr std::array kCases{ + // Default selectors exercise the SIMD path when CLAMP is clear. + TestCase{cdna5::kVPkMadI16Vop3p, 0x80007fffu, 0x00020002u, 0x00000000u, 0x80007fffu, 0u, 3u, + 1u}, + TestCase{cdna5::kVPkMadU16Vop3p, 0x0001ffffu, 0x00020002u, 0x00000000u, 0x0002ffffu, 0u, 3u, + 1u}, + TestCase{cdna5::kVPkMadI16Vop3p, 0x80007fffu, 0x00020002u, 0x00000000u, 0x0000fffeu, 0u, 3u, + 1u, false}, + TestCase{cdna5::kVPkMadU16Vop3p, 0x0001ffffu, 0x00020002u, 0x00000000u, 0x0002fffeu, 0u, 3u, + 1u, false}, + // Non-default selectors force scalar fallback and verify half selection. + TestCase{cdna5::kVPkMadI16Vop3p, 0x7fff8000u, 0x00020002u, 0x00000000u, 0x80007fffu, 7u, 0u, + 0u}, + TestCase{cdna5::kVPkMadU16Vop3p, 0xffff0001u, 0x00020002u, 0x00010000u, 0x0002ffffu, 3u, 0u, + 0u}, + TestCase{cdna5::kVPkMadI16Vop3p, 0x7fff8000u, 0x00020002u, 0x00000000u, 0x0000fffeu, 7u, 0u, + 0u, false}, + TestCase{cdna5::kVPkMadU16Vop3p, 0xffff0001u, 0x00020002u, 0x00010000u, 0x0002fffeu, 3u, 0u, + 0u, false}, + }; + for (uint32_t scalar = 0; scalar < 2; ++scalar) { + util::set_force_scalar_for_testing(scalar != 0); + for (const TestCase &test : kCases) { + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + const uint32_t base = wf->vgpr_alloc().base; + cu->write_vgpr(base, 0, test.src0); + cu->write_vgpr(base + 1, 0, test.src1); + cu->write_vgpr(base + 2, 0, test.src2); + const auto words = cdna5::build_vop3p(test.opcode, {.vdst = 3, + .opsel = test.opsel, + .opsel_hi_2 = test.opsel_hi_2, + .clamp = static_cast(test.clamp), + .src0 = 256, + .src1 = 257, + .src2 = 258, + .opsel_hi = test.opsel_hi}); + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA5); + ASSERT_NE(decoder, nullptr); + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + inst->execute(*inst, wf); + EXPECT_EQ(cu->read_vgpr(base + 3, 0), test.expected) << "scalar " << scalar; + } + } +} + TEST(Gfx1250ExecutionTest, Vop3CarryClampSaturatesVdstAndPreservesCarryBorrow) { ForceScalarGuard guard; struct TestCase { From 37721f2a297d5f6bcfcb6ba303a5c771976d72cf Mon Sep 17 00:00:00 2001 From: Sambhav Jain Date: Wed, 2 Sep 2026 23:18:57 +0000 Subject: [PATCH 11/16] [rocjitsu] Cover cleared MAD carry results Signed-off-by: Sambhav Jain --- .../rocjitsu/tests/cdna5_execution_test.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/emulation/rocjitsu/tests/cdna5_execution_test.cpp b/emulation/rocjitsu/tests/cdna5_execution_test.cpp index 21431d7928e8..65d567d93cdf 100644 --- a/emulation/rocjitsu/tests/cdna5_execution_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_execution_test.cpp @@ -379,14 +379,17 @@ TEST(Gfx1250ExecutionTest, Vop3Mad64ClampSaturatesAndPreservesCarryNormalAndDpp) uint32_t src1; uint64_t src2; uint64_t expected; + bool expected_carry; }; constexpr std::array kCases{ - TestCase{false, false, UINT32_MAX, UINT32_MAX, UINT64_MAX, UINT64_MAX}, - TestCase{false, true, UINT32_MAX, UINT32_MAX, UINT64_MAX, UINT64_MAX}, + TestCase{false, false, UINT32_MAX, UINT32_MAX, UINT64_MAX, UINT64_MAX, false}, + TestCase{false, true, UINT32_MAX, UINT32_MAX, UINT64_MAX, UINT64_MAX, true}, + TestCase{false, true, 2u, 3u, 4u, 10u, false}, TestCase{true, false, static_cast(INT32_MAX), static_cast(INT32_MAX), - static_cast(INT64_MAX), static_cast(INT64_MAX)}, + static_cast(INT64_MAX), static_cast(INT64_MAX), false}, TestCase{true, true, static_cast(INT32_MIN), static_cast(INT32_MAX), - static_cast(INT64_MIN), static_cast(INT64_MIN)}, + static_cast(INT64_MIN), static_cast(INT64_MIN), true}, + TestCase{true, true, static_cast(-2), 3u, 10u, 4u, false}, }; for (bool dpp : {false, true}) { @@ -403,7 +406,8 @@ TEST(Gfx1250ExecutionTest, Vop3Mad64ClampSaturatesAndPreservesCarryNormalAndDpp) cu->write_vgpr(base + 3, 0, static_cast(test.src2 >> 32)); cu->write_vgpr(base + 4, 0, 0u); cu->write_vgpr(base + 5, 0, 0u); - write_wave_sgpr(*cu, *wf, 2, 0u); + // Seed CO with a stale set bit so non-overflowing cases prove it is replaced. + write_wave_sgpr(*cu, *wf, 2, test.writes_carry ? 1u : 0u); if (!dpp) { std::array words; @@ -469,7 +473,7 @@ TEST(Gfx1250ExecutionTest, Vop3Mad64ClampSaturatesAndPreservesCarryNormalAndDpp) EXPECT_EQ(result, test.expected) << "signed " << test.is_signed << " co " << test.writes_carry << " dpp " << dpp; if (test.writes_carry) - EXPECT_EQ(read_wave_sgpr(*cu, *wf, 2) & 1u, 1u); + EXPECT_EQ(read_wave_sgpr(*cu, *wf, 2) & 1u, test.expected_carry); } } From 10a1349fd7c45fe871b35e0af99cd1a55b41ea24 Mon Sep 17 00:00:00 2001 From: Sambhav Jain Date: Wed, 2 Sep 2026 23:25:35 +0000 Subject: [PATCH 12/16] [rocjitsu] Exercise both MAD carry transitions Signed-off-by: Sambhav Jain --- emulation/rocjitsu/tests/cdna5_execution_test.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/emulation/rocjitsu/tests/cdna5_execution_test.cpp b/emulation/rocjitsu/tests/cdna5_execution_test.cpp index 65d567d93cdf..578a121e3d82 100644 --- a/emulation/rocjitsu/tests/cdna5_execution_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_execution_test.cpp @@ -406,8 +406,9 @@ TEST(Gfx1250ExecutionTest, Vop3Mad64ClampSaturatesAndPreservesCarryNormalAndDpp) cu->write_vgpr(base + 3, 0, static_cast(test.src2 >> 32)); cu->write_vgpr(base + 4, 0, 0u); cu->write_vgpr(base + 5, 0, 0u); - // Seed CO with a stale set bit so non-overflowing cases prove it is replaced. - write_wave_sgpr(*cu, *wf, 2, test.writes_carry ? 1u : 0u); + // Seed CO opposite to the expected bit so every case proves it is replaced. + write_wave_sgpr(*cu, *wf, 2, + test.writes_carry ? static_cast(!test.expected_carry) : 0u); if (!dpp) { std::array words; From 9b1bfe4f48d369ba233a5a30295963dcbb473713 Mon Sep 17 00:00:00 2001 From: Sambhav Jain Date: Wed, 2 Sep 2026 23:50:59 +0000 Subject: [PATCH 13/16] [rocjitsu] Propagate MAD64 clamp to shared paths Signed-off-by: Sambhav Jain --- .../rocjitsu/lib/python/amdisa/isa_profile.py | 2 + .../tests/test_generator_profile_gates.py | 4 ++ .../amdgpu/generated/shared/execute_shared.h | 8 ++++ .../isa/arch/amdgpu/shared/simd_glue.h | 4 ++ .../vop3_shift64_simd_correctness_test.cpp | 45 ++++++++++++++++++- 5 files changed, 61 insertions(+), 2 deletions(-) diff --git a/emulation/rocjitsu/lib/python/amdisa/isa_profile.py b/emulation/rocjitsu/lib/python/amdisa/isa_profile.py index 71a89447e17b..d576ffe43674 100644 --- a/emulation/rocjitsu/lib/python/amdisa/isa_profile.py +++ b/emulation/rocjitsu/lib/python/amdisa/isa_profile.py @@ -610,6 +610,7 @@ def integer_clamp_dtypes(self) -> dict[str, str]: 'V_MAD_I16': 'i16', 'V_MAD_I32_I16': 'i32', 'V_MAD_I32_I24': 'i32', + 'V_MAD_I64_I32': 'i64', 'V_MAD_CO_I64_I32': 'i64', 'V_MAD_CO_U64_U32': 'u64', 'V_MAD_NC_I64_I32': 'i64', @@ -617,6 +618,7 @@ def integer_clamp_dtypes(self) -> dict[str, str]: 'V_MAD_U16': 'u16', 'V_MAD_U32_U16': 'u32', 'V_MAD_U32_U24': 'u32', + 'V_MAD_U64_U32': 'u64', 'V_MQSAD_PK_U16_U8': 'u16', 'V_MQSAD_U32_U8': 'u32', 'V_MSAD_U8': 'u32', diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py index d4ec1c80934f..630a2a41dbdc 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py @@ -1792,6 +1792,10 @@ def test_vop3_mad_64_32_clamps_exact_result_without_changing_carry(): assert 'if (inst_.clamp)' in signed assert signed.index('carry |= 1ULL << lane') < signed.index('if (inst_.clamp)') + policy = Cdna4Profile().integer_clamp_dtypes + assert policy['V_MAD_U64_U32'] == 'u64' + assert policy['V_MAD_I64_I32'] == 'i64' + def test_vector_cmp_class_writes_explicit_sdst_mask(): body = gen_vector_cmp_class( diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h index 8045b615353e..69436c8dd764 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h @@ -13382,6 +13382,12 @@ inline void execute_v_mad_i64_i32_vop3([[maybe_unused]] Inst &inst, [[maybe_unus uint64_t result = static_cast(wide); if (wide < std::numeric_limits::min() || wide > std::numeric_limits::max()) carry |= 1ULL << lane; + if (inst.inst_.clamp) { + if (wide < std::numeric_limits::min()) + result = std::bit_cast(std::numeric_limits::min()); + else if (wide > std::numeric_limits::max()) + result = static_cast(std::numeric_limits::max()); + } sdwa::write_lane64(inst, wf, inst.vdst, lane, result); } commit_result(carry); @@ -13644,6 +13650,8 @@ inline void execute_v_mad_u64_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unus uint64_t result = static_cast(wide); if (wide > std::numeric_limits::max()) carry |= 1ULL << lane; + if (inst.inst_.clamp && wide > std::numeric_limits::max()) + result = std::numeric_limits::max(); sdwa::write_lane64(inst, wf, inst.vdst, lane, result); } commit_result(carry); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h index a1e141e02f82..1619bec00db1 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h @@ -3428,6 +3428,10 @@ template if (simd_force_scalar() || !inst.src0.simd_capable() || !inst.src1.simd_capable() || !inst.src2.simd_capable() || !inst.vdst.simd_capable()) return false; + if constexpr (requires { inst.inst_.clamp; }) { + if (inst.inst_.clamp) + return false; + } constexpr std::size_t W = util::native_width64; const uint64_t chunk_full = util::mask(static_cast(W)); const uint64_t exec = dpp::execution_lane_mask(inst, wf); diff --git a/emulation/rocjitsu/tests/simd_correctness/vop3_shift64_simd_correctness_test.cpp b/emulation/rocjitsu/tests/simd_correctness/vop3_shift64_simd_correctness_test.cpp index c5bc04c3d37b..ceac91458ed7 100644 --- a/emulation/rocjitsu/tests/simd_correctness/vop3_shift64_simd_correctness_test.cpp +++ b/emulation/rocjitsu/tests/simd_correctness/vop3_shift64_simd_correctness_test.cpp @@ -60,8 +60,10 @@ constexpr void vop3_encode(uint32_t op, uint32_t vdst, uint32_t src0, uint32_t s // VOP3_SDST_ENC: word0 also carries sdst[14:8]. constexpr void vop3_sdstenc_encode(uint32_t op, uint32_t vdst, uint32_t sdst, uint32_t src0, - uint32_t src1, uint32_t src2, uint32_t words[2]) { - words[0] = (vdst & 0xFF) | ((sdst & 0x7F) << 8) | ((op & 0x3FF) << 16) | (0x34u << 26); + uint32_t src1, uint32_t src2, uint32_t words[2], + uint32_t clamp = 0) { + words[0] = (vdst & 0xFF) | ((sdst & 0x7F) << 8) | ((clamp & 0x1) << 15) | ((op & 0x3FF) << 16) | + (0x34u << 26); words[1] = (src0 & 0x1FF) | ((src1 & 0x1FF) << 9) | ((src2 & 0x1FF) << 18); } @@ -316,6 +318,45 @@ TEST(Vop3Shift64SimdCorrectness, MadWide64) { check_mad64("v_mad_i64_i32_vop3", 489, /*is_signed=*/true, 0xA5A5'F0F0'1234'8001ULL); } +TEST(Vop3Shift64SimdCorrectness, MadWide64ClampSaturatesAndPreservesCarry) { + ForceScalarGuard gate_guard; + for (bool is_signed : {false, true}) { + for (bool force_scalar : {false, true}) { + util::set_force_scalar_for_testing(force_scalar); + Fixture fx; + ASSERT_NE(fx.cu, nullptr); + ASSERT_NE(fx.wf, nullptr); + const uint32_t sb = fx.wf->sgpr_alloc().base; + ASSERT_EQ(sb % 2u, 0u); + const uint32_t opcode = is_signed ? 489u : 488u; + uint32_t words[2] = {0u, 0u}; + vop3_sdstenc_encode(opcode, kDstVgpr, sb, /*src0=*/256, /*src1=*/258, /*src2=*/260, words, + /*clamp=*/1); + std::unique_ptr inst(decode_valid(*fx.decoder, words)); + ASSERT_NE(inst, nullptr); + const uint32_t vb = fx.wf->vgpr_alloc().base; + if (is_signed) { + fx.cu->write_vgpr(vb, 0, static_cast(INT32_MIN)); + fx.cu->write_vgpr(vb + 2, 0, static_cast(INT32_MAX)); + fx.write64(vb + 4, 0, static_cast(INT64_MIN)); + } else { + fx.cu->write_vgpr(vb, 0, UINT32_MAX); + fx.cu->write_vgpr(vb + 2, 0, UINT32_MAX); + fx.write64(vb + 4, 0, UINT64_MAX); + } + fx.write64(vb + kDstVgpr, 0, 0u); + fx.write_sgpr64(sb, 0u); + fx.wf->set_exec(1u); + fx.cu->execute_instruction(inst.get(), *fx.wf); + EXPECT_EQ(fx.read64(vb + kDstVgpr, 0), + is_signed ? static_cast(INT64_MIN) : UINT64_MAX) + << "signed " << is_signed << " scalar " << force_scalar; + EXPECT_EQ(fx.read_sgpr64(sb) & 1u, 1u) + << "signed " << is_signed << " scalar " << force_scalar; + } + } +} + TEST(Vop3Shift64SimdCorrectness, RevShiftsFullExec) { if constexpr (!util::has_stdx_simd) { GTEST_SKIP() << " unavailable — scalar fallback in use"; From 651cb0fd72c82405bd3894c51ade25b235d288c5 Mon Sep 17 00:00:00 2001 From: Sambhav Jain Date: Wed, 2 Sep 2026 23:59:29 +0000 Subject: [PATCH 14/16] [rocjitsu] Cover signed MAD64 positive clamp Signed-off-by: Sambhav Jain --- .../vop3_shift64_simd_correctness_test.cpp | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/emulation/rocjitsu/tests/simd_correctness/vop3_shift64_simd_correctness_test.cpp b/emulation/rocjitsu/tests/simd_correctness/vop3_shift64_simd_correctness_test.cpp index ceac91458ed7..301472be9f46 100644 --- a/emulation/rocjitsu/tests/simd_correctness/vop3_shift64_simd_correctness_test.cpp +++ b/emulation/rocjitsu/tests/simd_correctness/vop3_shift64_simd_correctness_test.cpp @@ -320,7 +320,21 @@ TEST(Vop3Shift64SimdCorrectness, MadWide64) { TEST(Vop3Shift64SimdCorrectness, MadWide64ClampSaturatesAndPreservesCarry) { ForceScalarGuard gate_guard; - for (bool is_signed : {false, true}) { + struct ClampCase { + bool is_signed; + uint32_t src0; + uint32_t src1; + uint64_t src2; + uint64_t expected; + }; + constexpr std::array kCases{ + ClampCase{false, UINT32_MAX, UINT32_MAX, UINT64_MAX, UINT64_MAX}, + ClampCase{true, static_cast(INT32_MIN), static_cast(INT32_MAX), + static_cast(INT64_MIN), static_cast(INT64_MIN)}, + ClampCase{true, static_cast(INT32_MAX), static_cast(INT32_MAX), + static_cast(INT64_MAX), static_cast(INT64_MAX)}, + }; + for (const ClampCase &test : kCases) { for (bool force_scalar : {false, true}) { util::set_force_scalar_for_testing(force_scalar); Fixture fx; @@ -328,31 +342,24 @@ TEST(Vop3Shift64SimdCorrectness, MadWide64ClampSaturatesAndPreservesCarry) { ASSERT_NE(fx.wf, nullptr); const uint32_t sb = fx.wf->sgpr_alloc().base; ASSERT_EQ(sb % 2u, 0u); - const uint32_t opcode = is_signed ? 489u : 488u; + const uint32_t opcode = test.is_signed ? 489u : 488u; uint32_t words[2] = {0u, 0u}; vop3_sdstenc_encode(opcode, kDstVgpr, sb, /*src0=*/256, /*src1=*/258, /*src2=*/260, words, /*clamp=*/1); std::unique_ptr inst(decode_valid(*fx.decoder, words)); ASSERT_NE(inst, nullptr); const uint32_t vb = fx.wf->vgpr_alloc().base; - if (is_signed) { - fx.cu->write_vgpr(vb, 0, static_cast(INT32_MIN)); - fx.cu->write_vgpr(vb + 2, 0, static_cast(INT32_MAX)); - fx.write64(vb + 4, 0, static_cast(INT64_MIN)); - } else { - fx.cu->write_vgpr(vb, 0, UINT32_MAX); - fx.cu->write_vgpr(vb + 2, 0, UINT32_MAX); - fx.write64(vb + 4, 0, UINT64_MAX); - } + fx.cu->write_vgpr(vb, 0, test.src0); + fx.cu->write_vgpr(vb + 2, 0, test.src1); + fx.write64(vb + 4, 0, test.src2); fx.write64(vb + kDstVgpr, 0, 0u); fx.write_sgpr64(sb, 0u); fx.wf->set_exec(1u); fx.cu->execute_instruction(inst.get(), *fx.wf); - EXPECT_EQ(fx.read64(vb + kDstVgpr, 0), - is_signed ? static_cast(INT64_MIN) : UINT64_MAX) - << "signed " << is_signed << " scalar " << force_scalar; + EXPECT_EQ(fx.read64(vb + kDstVgpr, 0), test.expected) + << "signed " << test.is_signed << " scalar " << force_scalar; EXPECT_EQ(fx.read_sgpr64(sb) & 1u, 1u) - << "signed " << is_signed << " scalar " << force_scalar; + << "signed " << test.is_signed << " scalar " << force_scalar; } } } From 178b34fbefdf26e51b793aa49f93d165391cd59a Mon Sep 17 00:00:00 2001 From: Sambhav Jain Date: Thu, 3 Sep 2026 00:36:02 +0000 Subject: [PATCH 15/16] [rocjitsu] Avoid non-standard MAD64 intermediates Compute the exact 32x32 product in 64 bits and detect signed or unsigned addition overflow explicitly. This preserves clamp and carry semantics while compiling cleanly with GCC -Wpedantic. Also brace the associated carry assertion to satisfy GCC's dangling-else warning under -Werror. Signed-off-by: Sambhav Jain --- .../amdisa/codegen/execute/vector_special.py | 31 +++++------ .../tests/test_generator_profile_gates.py | 23 ++++---- .../generated/cdna5/vop3_exec_ternary.cpp | 46 ++++++++-------- .../amdgpu/generated/shared/execute_shared.h | 54 ++++++++++--------- .../rocjitsu/tests/cdna5_execution_test.cpp | 3 +- 5 files changed, 81 insertions(+), 76 deletions(-) diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py index 039126ba1264..c33f7308205a 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py @@ -736,22 +736,19 @@ def gen_vector_mad_64_32( f' int64_t s1 = static_cast(amdgpu::RegisterAccess(wf).read_lane({src[1]}, lane));' ) L.append( - f' int64_t s2 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64({src[2]}, lane));' + f' uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64({src[2]}, lane);' ) - L.append(' __int128 wide = static_cast<__int128>(s0) * s1 + s2;') - L.append(' uint64_t result = static_cast(wide);') + L.append(' uint64_t product = static_cast(s0 * s1);') + L.append(' uint64_t result = product + s2;') + if writes_carry or integer_clamp: + L.append(' bool overflow = amdgpu::signed_add_overflows(product, s2);') if writes_carry: - L.append( - ' if (wide < std::numeric_limits::min() || wide > std::numeric_limits::max())' - ) + L.append(' if (overflow)') L.append(' carry |= 1ULL << lane;') if integer_clamp: - L.append(' if (inst_.clamp) {') + L.append(' if (inst_.clamp && overflow) {') L.append( - ' if (wide < std::numeric_limits::min()) result = std::bit_cast(std::numeric_limits::min());' - ) - L.append( - ' else if (wide > std::numeric_limits::max()) result = static_cast(std::numeric_limits::max());' + ' result = (product & (1ULL << 63)) ? std::bit_cast(std::numeric_limits::min()) : static_cast(std::numeric_limits::max());' ) L.append(' }') L.append( @@ -767,16 +764,16 @@ def gen_vector_mad_64_32( L.append( f' uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64({src[2]}, lane);' ) - L.append( - ' unsigned __int128 wide = static_cast(s0) * s1 + s2;' - ) - L.append(' uint64_t result = static_cast(wide);') + L.append(' uint64_t product = s0 * s1;') + L.append(' uint64_t result = product + s2;') + if writes_carry or integer_clamp: + L.append(' bool overflow = result < product;') if writes_carry: - L.append(' if (wide > std::numeric_limits::max())') + L.append(' if (overflow)') L.append(' carry |= 1ULL << lane;') if integer_clamp: L.append( - ' if (inst_.clamp && wide > std::numeric_limits::max()) result = std::numeric_limits::max();' + ' if (inst_.clamp && overflow) result = std::numeric_limits::max();' ) L.append( f' amdgpu::RegisterAccess(wf).write_lane64({dst[0]}, lane, result);' diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py index 630a2a41dbdc..d7e492e81222 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py @@ -1754,8 +1754,8 @@ def test_vop3_mad_u64_u32_writes_explicit_sdst_carry(): body = gen_vector_mad_64_32(['vdst', 'sdst'], ['src0', 'src1', 'src2'], 'u64') assert 'uint64_t carry = 0;' in body - assert 'unsigned __int128 wide' in body - assert 'if (wide > std::numeric_limits::max())' in body + assert 'uint64_t product = s0 * s1;' in body + assert 'bool overflow = result < product;' in body assert 'carry |= 1ULL << lane;' in body assert 'amdgpu::write_wave_mask_scalar(sdst, wf, carry);' in body assert 'wf.set_vcc' not in body @@ -1784,13 +1784,15 @@ def test_vop3_mad_64_32_clamps_exact_result_without_changing_carry(): integer_clamp=True, ) - assert 'unsigned __int128 wide' in unsigned - assert 'inst_.clamp && wide > std::numeric_limits::max()' in unsigned - assert unsigned.index('if (wide >') < unsigned.index('if (inst_.clamp') - assert '__int128 wide' in signed - assert 'wide < std::numeric_limits::min()' in signed - assert 'if (inst_.clamp)' in signed - assert signed.index('carry |= 1ULL << lane') < signed.index('if (inst_.clamp)') + assert 'bool overflow = result < product;' in unsigned + assert 'inst_.clamp && overflow' in unsigned + assert unsigned.index('carry |= 1ULL << lane') < unsigned.index('inst_.clamp') + assert 'amdgpu::signed_add_overflows(product, s2)' in signed + assert 'inst_.clamp && overflow' in signed + assert '(product & (1ULL << 63))' in signed + assert signed.index('carry |= 1ULL << lane') < signed.index('inst_.clamp') + assert '__int128' not in unsigned + assert '__int128' not in signed policy = Cdna4Profile().integer_clamp_dtypes assert policy['V_MAD_U64_U32'] == 'u64' @@ -4159,7 +4161,8 @@ def generated_body(name: str, enc_name: str, *, result_writer=None) -> str: ('V_MAD_CO_I64_I32', 'VOP3_SDST_ENC', 'commit_result'), ): body = generated_body(name, enc_name, result_writer=result_writer) - assert '__int128 wide' in body + assert 'bool overflow' in body + assert '__int128' not in body assert 'inst_.clamp' in body for name, helper in ( diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp index be61e9361b45..03066b01a3df 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp @@ -3972,9 +3972,10 @@ void VMadNcU64U32Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(src2, lane); - unsigned __int128 wide = static_cast(s0) * s1 + s2; - uint64_t result = static_cast(wide); - if (inst_.clamp && wide > std::numeric_limits::max()) + uint64_t product = s0 * s1; + uint64_t result = product + s2; + bool overflow = result < product; + if (inst_.clamp && overflow) result = std::numeric_limits::max(); amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, result); } @@ -4001,9 +4002,10 @@ RJ_NOINLINE void VMadNcU64U32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane(src0, lane); uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane(src1, lane); uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(src2, lane); - unsigned __int128 wide = static_cast(s0) * s1 + s2; - uint64_t result = static_cast(wide); - if (inst_.clamp && wide > std::numeric_limits::max()) + uint64_t product = s0 * s1; + uint64_t result = product + s2; + bool overflow = result < product; + if (inst_.clamp && overflow) result = std::numeric_limits::max(); amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, result); } @@ -4021,14 +4023,14 @@ void VMadNcI64I32Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; int64_t s0 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); int64_t s1 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - int64_t s2 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src2, lane)); - __int128 wide = static_cast<__int128>(s0) * s1 + s2; - uint64_t result = static_cast(wide); - if (inst_.clamp) { - if (wide < std::numeric_limits::min()) - result = std::bit_cast(std::numeric_limits::min()); - else if (wide > std::numeric_limits::max()) - result = static_cast(std::numeric_limits::max()); + uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(src2, lane); + uint64_t product = static_cast(s0 * s1); + uint64_t result = product + s2; + bool overflow = amdgpu::signed_add_overflows(product, s2); + if (inst_.clamp && overflow) { + result = (product & (1ULL << 63)) + ? std::bit_cast(std::numeric_limits::min()) + : static_cast(std::numeric_limits::max()); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, result); } @@ -4054,14 +4056,14 @@ RJ_NOINLINE void VMadNcI64I32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) continue; int64_t s0 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); int64_t s1 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); - int64_t s2 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(src2, lane)); - __int128 wide = static_cast<__int128>(s0) * s1 + s2; - uint64_t result = static_cast(wide); - if (inst_.clamp) { - if (wide < std::numeric_limits::min()) - result = std::bit_cast(std::numeric_limits::min()); - else if (wide > std::numeric_limits::max()) - result = static_cast(std::numeric_limits::max()); + uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(src2, lane); + uint64_t product = static_cast(s0 * s1); + uint64_t result = product + s2; + bool overflow = amdgpu::signed_add_overflows(product, s2); + if (inst_.clamp && overflow) { + result = (product & (1ULL << 63)) + ? std::bit_cast(std::numeric_limits::min()) + : static_cast(std::numeric_limits::max()); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, result); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h index 69436c8dd764..2561e25067d5 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h @@ -13200,16 +13200,16 @@ inline void execute_v_mad_co_i64_i32_vop3([[maybe_unused]] Inst &inst, continue; int64_t s0 = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); int64_t s1 = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - int64_t s2 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src2, lane)); - __int128 wide = static_cast<__int128>(s0) * s1 + s2; - uint64_t result = static_cast(wide); - if (wide < std::numeric_limits::min() || wide > std::numeric_limits::max()) + uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(inst.src2, lane); + uint64_t product = static_cast(s0 * s1); + uint64_t result = product + s2; + bool overflow = amdgpu::signed_add_overflows(product, s2); + if (overflow) carry |= 1ULL << lane; - if (inst.inst_.clamp) { - if (wide < std::numeric_limits::min()) - result = std::bit_cast(std::numeric_limits::min()); - else if (wide > std::numeric_limits::max()) - result = static_cast(std::numeric_limits::max()); + if (inst.inst_.clamp && overflow) { + result = (product & (1ULL << 63)) + ? std::bit_cast(std::numeric_limits::min()) + : static_cast(std::numeric_limits::max()); } sdwa::write_lane64(inst, wf, inst.vdst, lane, result); } @@ -13228,11 +13228,12 @@ inline void execute_v_mad_co_u64_u32_vop3([[maybe_unused]] Inst &inst, uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane); uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane); uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(inst.src2, lane); - unsigned __int128 wide = static_cast(s0) * s1 + s2; - uint64_t result = static_cast(wide); - if (wide > std::numeric_limits::max()) + uint64_t product = s0 * s1; + uint64_t result = product + s2; + bool overflow = result < product; + if (overflow) carry |= 1ULL << lane; - if (inst.inst_.clamp && wide > std::numeric_limits::max()) + if (inst.inst_.clamp && overflow) result = std::numeric_limits::max(); sdwa::write_lane64(inst, wf, inst.vdst, lane, result); } @@ -13377,16 +13378,16 @@ inline void execute_v_mad_i64_i32_vop3([[maybe_unused]] Inst &inst, [[maybe_unus continue; int64_t s0 = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); int64_t s1 = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); - int64_t s2 = std::bit_cast(amdgpu::RegisterAccess(wf).read_lane64(inst.src2, lane)); - __int128 wide = static_cast<__int128>(s0) * s1 + s2; - uint64_t result = static_cast(wide); - if (wide < std::numeric_limits::min() || wide > std::numeric_limits::max()) + uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(inst.src2, lane); + uint64_t product = static_cast(s0 * s1); + uint64_t result = product + s2; + bool overflow = amdgpu::signed_add_overflows(product, s2); + if (overflow) carry |= 1ULL << lane; - if (inst.inst_.clamp) { - if (wide < std::numeric_limits::min()) - result = std::bit_cast(std::numeric_limits::min()); - else if (wide > std::numeric_limits::max()) - result = static_cast(std::numeric_limits::max()); + if (inst.inst_.clamp && overflow) { + result = (product & (1ULL << 63)) + ? std::bit_cast(std::numeric_limits::min()) + : static_cast(std::numeric_limits::max()); } sdwa::write_lane64(inst, wf, inst.vdst, lane, result); } @@ -13646,11 +13647,12 @@ inline void execute_v_mad_u64_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unus uint64_t s0 = amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane); uint64_t s1 = amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane); uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(inst.src2, lane); - unsigned __int128 wide = static_cast(s0) * s1 + s2; - uint64_t result = static_cast(wide); - if (wide > std::numeric_limits::max()) + uint64_t product = s0 * s1; + uint64_t result = product + s2; + bool overflow = result < product; + if (overflow) carry |= 1ULL << lane; - if (inst.inst_.clamp && wide > std::numeric_limits::max()) + if (inst.inst_.clamp && overflow) result = std::numeric_limits::max(); sdwa::write_lane64(inst, wf, inst.vdst, lane, result); } diff --git a/emulation/rocjitsu/tests/cdna5_execution_test.cpp b/emulation/rocjitsu/tests/cdna5_execution_test.cpp index 578a121e3d82..26b760e0c48f 100644 --- a/emulation/rocjitsu/tests/cdna5_execution_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_execution_test.cpp @@ -473,8 +473,9 @@ TEST(Gfx1250ExecutionTest, Vop3Mad64ClampSaturatesAndPreservesCarryNormalAndDpp) (static_cast(cu->read_vgpr(base + 5, 0)) << 32); EXPECT_EQ(result, test.expected) << "signed " << test.is_signed << " co " << test.writes_carry << " dpp " << dpp; - if (test.writes_carry) + if (test.writes_carry) { EXPECT_EQ(read_wave_sgpr(*cu, *wf, 2) & 1u, test.expected_carry); + } } } From b93d19685ef2188739ce1d32415162e9185359bc Mon Sep 17 00:00:00 2001 From: Sambhav Jain Date: Thu, 3 Sep 2026 03:48:00 +0000 Subject: [PATCH 16/16] [rocjitsu] Cover both signed MAD64 clamp directions Exercise positive overflow and negative underflow in both CDNA5 NC and CO execute bodies. This protects each generated saturation branch from direction-specific drift. Signed-off-by: Sambhav Jain --- emulation/rocjitsu/tests/cdna5_execution_test.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/emulation/rocjitsu/tests/cdna5_execution_test.cpp b/emulation/rocjitsu/tests/cdna5_execution_test.cpp index 26b760e0c48f..14fbba7473c4 100644 --- a/emulation/rocjitsu/tests/cdna5_execution_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_execution_test.cpp @@ -387,6 +387,10 @@ TEST(Gfx1250ExecutionTest, Vop3Mad64ClampSaturatesAndPreservesCarryNormalAndDpp) TestCase{false, true, 2u, 3u, 4u, 10u, false}, TestCase{true, false, static_cast(INT32_MAX), static_cast(INT32_MAX), static_cast(INT64_MAX), static_cast(INT64_MAX), false}, + TestCase{true, false, static_cast(INT32_MIN), static_cast(INT32_MAX), + static_cast(INT64_MIN), static_cast(INT64_MIN), false}, + TestCase{true, true, static_cast(INT32_MAX), static_cast(INT32_MAX), + static_cast(INT64_MAX), static_cast(INT64_MAX), true}, TestCase{true, true, static_cast(INT32_MIN), static_cast(INT32_MAX), static_cast(INT64_MIN), static_cast(INT64_MIN), true}, TestCase{true, true, static_cast(-2), 3u, 10u, 4u, false},