diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py index 232a2bbcf4d2..494c3ab091f5 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py @@ -6148,6 +6148,10 @@ def _semantic_reg_class(opnd: Operand) -> RegClass: mode_sensitive_f16_dst=not cls.startswith('scalar_'), mask_result_writer=result_writer, ) + inst_fields = getattr(self, '_current_inst_fields', set()) + integer_clamp_dtype = profile.integer_clamp_dtypes.get(inst.name) + if is_vop3 and 'clamp' in inst_fields and integer_clamp_dtype: + lctx.integer_saturation_dtype = integer_clamp_dtype if cls == 'vector_cmp': # V_CMP writes a fresh wave mask initialized to zero, so false # lanes can remain clear without emitting redundant bit clears. @@ -6266,17 +6270,13 @@ def _semantic_reg_class(opnd: Operand) -> RegClass: ), 'V_MAD_I16': ( 3, - ( - 'int32_t a = static_cast(s0);', - 'int32_t b = static_cast(s1);', - 'int32_t c = static_cast(s2);', - ), - 'static_cast(static_cast(a * b + c))', + (), + 'amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)', ), 'V_MAD_U16': ( 3, (), - '(s0 * s1 + s2) & 0xffffu', + 'amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)', ), } if is_true16_vop3 and inst.name in true16_special_vop3_ops: diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py index 82f84476971b..a0c25518bde8 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py @@ -208,13 +208,21 @@ def _register_handlers() -> None: c.dst_ops, c.src_ops, c.op, c.profile.uses_vgpr_msb_indexing ) DISPATCH['vector_qsad'] = lambda c: gen_vector_qsad( - c.dst_ops, c.src_ops, c.op, c.profile.uses_vgpr_msb_indexing + c.dst_ops, + c.src_ops, + c.op, + c.profile.uses_vgpr_msb_indexing, + c.is_vop3 and c.inst.name in c.profile.integer_clamp_dtypes, ) DISPATCH['vector_trig_preop'] = lambda c: gen_vector_trig_preop( c.dst_ops, c.src_ops, c.is_vop3, c.has_abs ) DISPATCH['vector_mad_64_32'] = lambda c: gen_vector_mad_64_32( - c.dst_ops, c.src_ops, c.dtype, c.result_writer + c.dst_ops, + c.src_ops, + c.dtype, + c.result_writer, + c.is_vop3 and c.inst.name in c.profile.integer_clamp_dtypes, ) DISPATCH['vector_mad_32_16'] = lambda c: gen_vector_mad_32_16( c.dst_ops, c.src_ops, c.dtype, c.is_vop3 @@ -309,6 +317,7 @@ def _register_handlers() -> None: c.dtype, op_sel_hi_2_expr=c.op_sel_hi_2_expr, opsel_exprs=c.opsel_exprs, + integer_clamp=c.inst.name in c.profile.integer_clamp_dtypes, ) DISPATCH['pk_fmac_vop2'] = lambda c: ( gen_pk_fmac_vop3(c.dst_ops, c.src_ops) diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py index 97bb39d5e866..82eb15802e4c 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py @@ -179,8 +179,16 @@ def gen_pk_binop( ), } lo_expr, hi_expr = i_op_map[op] - L.append(f' uint16_t rlo = static_cast({lo_expr});') - L.append(f' uint16_t rhi = static_cast({hi_expr});') + if op in ('add', 'sub'): + L.append( + f' uint16_t rlo = static_cast(amdgpu::vop3_integer_{op}(a_lo, b_lo, inst_.clamp));' + ) + L.append( + f' uint16_t rhi = static_cast(amdgpu::vop3_integer_{op}(a_hi, b_hi, inst_.clamp));' + ) + else: + L.append(f' uint16_t rlo = static_cast({lo_expr});') + L.append(f' uint16_t rhi = static_cast({hi_expr});') L.append( f' amdgpu::RegisterAccess(wf).write_lane({d}, lane, static_cast(rlo) | (static_cast(rhi) << 16));' ) @@ -216,8 +224,16 @@ def gen_pk_binop( ), } lo_expr, hi_expr = u_op_map[op] - L.append(f' uint16_t rlo = static_cast({lo_expr});') - L.append(f' uint16_t rhi = static_cast({hi_expr});') + if op in ('add', 'sub'): + L.append( + f' uint16_t rlo = static_cast(amdgpu::vop3_integer_{op}(a_lo, b_lo, inst_.clamp));' + ) + L.append( + f' uint16_t rhi = static_cast(amdgpu::vop3_integer_{op}(a_hi, b_hi, inst_.clamp));' + ) + else: + L.append(f' uint16_t rlo = static_cast({lo_expr});') + L.append(f' uint16_t rhi = static_cast({hi_expr});') L.append( f' amdgpu::RegisterAccess(wf).write_lane({d}, lane, static_cast(rlo) | (static_cast(rhi) << 16));' ) @@ -233,6 +249,7 @@ def gen_pk_ternary( dtype: str | None, op_sel_hi_2_expr: str = '', opsel_exprs: tuple[str, str] = ('', ''), + integer_clamp: bool = False, ) -> str: """Generate packed 16-bit ternary op (V_PK_FMA_F16, V_PK_MAD_I16, etc.).""" d, s0, s1, s2 = dst[0], src[0], src[1], src[2] @@ -416,6 +433,17 @@ def gen_pk_ternary( L.append( ' uint16_t rhi = static_cast(std::max(std::max(a_hi, b_hi), c_hi));' ) + elif op == 'mad' and integer_clamp: + L.append( + ' uint16_t rlo = amdgpu::vop3_integer_mad(' + 'static_cast(a_lo), static_cast(b_lo), ' + 'static_cast(c_lo), inst_.clamp);' + ) + L.append( + ' uint16_t rhi = amdgpu::vop3_integer_mad(' + 'static_cast(a_hi), static_cast(b_hi), ' + 'static_cast(c_hi), inst_.clamp);' + ) else: L.append( ' uint16_t rlo = static_cast(static_cast(a_lo) * b_lo + c_lo);' @@ -478,6 +506,15 @@ def gen_pk_ternary( elif op in ('max3', 'maximum3'): L.append(' uint16_t rlo = std::max(std::max(a_lo, b_lo), c_lo);') L.append(' uint16_t rhi = std::max(std::max(a_hi, b_hi), c_hi);') + elif op == 'mad' and integer_clamp: + L.append( + ' uint16_t rlo = amdgpu::vop3_integer_mad(' + 'a_lo, b_lo, c_lo, inst_.clamp);' + ) + L.append( + ' uint16_t rhi = amdgpu::vop3_integer_mad(' + 'a_hi, b_hi, c_hi, inst_.clamp);' + ) else: L.append( ' uint16_t rlo = static_cast(static_cast(a_lo) * b_lo + c_lo);' diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py index 8aa702220893..6146ab8752e9 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py @@ -128,6 +128,7 @@ class LoweringContext: vector_sgpr_once: bool = False clear_false_lane_mask_writes: bool = True mode_sensitive_f16_dst: bool = True + integer_saturation_dtype: str | None = None _INFIX_OPS: dict[SemaNodeKind, str] = { @@ -537,6 +538,12 @@ def _lower_expr(node: SemaNode, ctx: LoweringContext) -> str: if kind == SemaNodeKind.ID: return _lower_id(node, ctx) + if ( + kind in (SemaNodeKind.ADD, SemaNodeKind.SUB) + and ctx.integer_saturation_dtype is not None + ): + return _lower_saturating_integer_binary(node, ctx) + if kind in _INFIX_OPS: if (expr := _lower_less_greater_once(node, ctx)) is not None: return expr @@ -727,6 +734,33 @@ def _lower_expr(node: SemaNode, ctx: LoweringContext) -> str: raise ValueError(f'Unhandled SemaNodeKind in lowering: {kind.name}') +def _lower_saturating_integer_binary(node: SemaNode, ctx: LoweringContext) -> str: + """Lower VOP3 integer add/sub with optional CLAMP saturation.""" + dtype = ctx.integer_saturation_dtype + if dtype not in ('i16', 'u16', 'i32', 'u32', 'u64'): + raise ValueError(f'Unsupported integer saturation type: {dtype}') + + child_ctx = replace(ctx, integer_saturation_dtype=None) + if node.kind == SemaNodeKind.ADD and node.children[0].kind == SemaNodeKind.MUL: + product = node.children[0] + lhs = _lower_expr(product.children[0], child_ctx) + rhs = _lower_expr(product.children[1], child_ctx) + addend = _lower_expr(node.children[1], child_ctx) + cpp_type = f'{"int" if dtype.startswith("i") else "uint"}{dtype[1:]}_t' + return ( + f'amdgpu::vop3_integer_mad<{cpp_type}, {dtype[1:]}>(' + f'{lhs}, {rhs}, {addend}, inst_.clamp)' + ) + + lhs = _lower_expr(node.children[0], child_ctx) + rhs = _lower_expr(node.children[1], child_ctx) + operation = 'add' if node.kind == SemaNodeKind.ADD else 'sub' + cpp_type = f'{"int" if dtype.startswith("i") else "uint"}{dtype[1:]}_t' + return ( + f'amdgpu::vop3_integer_{operation}<{cpp_type}>(' f'{lhs}, {rhs}, inst_.clamp)' + ) + + def _lower_lit(node: SemaNode) -> str: """Lower a literal value with appropriate C++ suffix.""" val = node.lit_value or '0' @@ -1729,6 +1763,66 @@ def _lower_call(node: SemaNode, ctx: LoweringContext) -> str: args = [_lower_expr(c, ctx) for c in node.children[1:]] args_str = ', '.join(args) + add_minmax_calls = { + 'add_max_i32': ('int32_t', 'true'), + 'add_max_u32': ('uint32_t', 'true'), + 'add_min_i32': ('int32_t', 'false'), + 'add_min_u32': ('uint32_t', 'false'), + } + if ctx.integer_saturation_dtype is not None and callee in add_minmax_calls: + cpp_type, select_max = add_minmax_calls[callee] + return ( + f'amdgpu::vop3_integer_add_minmax<{cpp_type}, {select_max}>(' f'{args_str})' + ) + + saturating_calls = { + 'mul_i24': ('int32_t', 24, False), + 'mul_u24': ('uint32_t', 24, False), + 'mad_i24': ('int32_t', 24, True), + 'mad_u24': ('uint32_t', 24, True), + 'mad_lo_u16': ('uint16_t', 16, True), + } + if ctx.integer_saturation_dtype is not None and callee in saturating_calls: + cpp_type, source_bits, has_addend = saturating_calls[callee] + helper = 'vop3_integer_mad' if has_addend else 'vop3_integer_mul' + return ( + f'amdgpu::{helper}<{cpp_type}, {source_bits}>(' f'{args_str}, inst_.clamp)' + ) + + if ctx.integer_saturation_dtype == 'u32' and callee in ( + 'sad_hi_u8', + 'sad_u8', + 'sad_u16', + 'sad_u32', + 'msad_u8', + ): + return f'amdgpu::vop3_integer_{callee}({args_str}, inst_.clamp)' + + if ctx.integer_saturation_dtype == 'u32' and callee in ( + 'add_co', + 'sub_co', + 'addc_co', + 'subbc_co', + ): + if callee in ('add_co', 'addc_co'): + terms = ' + '.join(f'static_cast({arg})' for arg in args) + return ( + f'[&]() {{ uint64_t w = {terms};' + ' if (w > 0xFFFFFFFFULL) vcc |= (1ULL << lane);' + ' else vcc &= ~(1ULL << lane);' + ' return inst_.clamp && w > 0xFFFFFFFFULL ? UINT32_MAX' + ' : static_cast(w); }()' + ) + minuend, subtrahend, *borrow = args + borrow_expr = f' + static_cast({borrow[0]})' if borrow else '' + return ( + f'[&]() {{ uint64_t a = static_cast({minuend}),' + f' b = static_cast({subtrahend}){borrow_expr};' + ' if (a < b) vcc |= (1ULL << lane);' + ' else vcc &= ~(1ULL << lane);' + ' return inst_.clamp && a < b ? 0u : static_cast(a - b); }()' + ) + pseudo_scalar_operations = { 'exp2': 'EXP2', 'log2': 'LOG2', diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/simd_codegen.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/simd_codegen.py index 7391f850187e..5dde1b5ee1a2 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/simd_codegen.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/simd_codegen.py @@ -1735,10 +1735,12 @@ def _build_simd_vopc_vop3_f64() -> dict[str, str]: # # 16-bit forms compute a 32-bit add/sub and mask the low 16 bits, matching the # scalar body's `uint32_t(uint16_t(int16_t(low16(a)+low16(b))))` (or the u16 -# variant) — both reduce to `(a + b) & 0xFFFFu` / `(a - b) & 0xFFFFu` because +# variant) — both reduce to `(a + b) & 0xFFFFu` / `(a - b) & 0xFFFFu` when +# CLAMP is clear because # unsigned 32-bit wrap-around at the low 16 bits is identical to signed/unsigned # 16-bit wrap. 32-bit forms use the wrap-around add/sub on uint32 lanes; -# signed-vs-unsigned wraps the same way. +# signed-vs-unsigned wraps the same way. The shared glue falls back to scalar +# execution when CLAMP requests saturating arithmetic. SIMD_VOP3_BINARY_INT_EXTRA: dict[str, tuple[str, str]] = { # v_bcnt_u32_b32: VOP3-only (no VOP1 twin). D = CountOneBits(S0) + S1 -- the # second source is an accumulator, so this is binary, not unary. Keep the diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py index b163d9f96ab6..c33f7308205a 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/vector_special.py @@ -496,7 +496,11 @@ def gen_vector_perm_pk16( def gen_vector_qsad( - dst: list[str], src: list[str], op: str | None, uses_vgpr_msb_indexing: bool + dst: list[str], + src: list[str], + op: str | None, + uses_vgpr_msb_indexing: bool, + integer_clamp: bool = False, ) -> str: """Generate QSAD/MQSAD four-window sum-of-absolute-difference ops.""" is_packed = op in ('sad_pk_u16', 'msad_pk_u16') @@ -553,17 +557,27 @@ def gen_vector_qsad( L.append(' sum += a > b ? a - b : b - a;') L.extend([' }']) if is_packed: + value_expr = ( + 'inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)' + if integer_clamp + else 'value & 0xffffu' + ) L.extend( [ ' uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu);', - ' packed_result |= static_cast(value & 0xffffu) << (window * 16);', + f' packed_result |= static_cast({value_expr}) << (window * 16);', ] ) else: + value_expr = ( + 'amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)' + if integer_clamp + else 'accum[window] + sum' + ) L.extend( [ f' dst_word.encoding_value_ = {dst[0]}.encoding_value_ + static_cast(window);', - ' amdgpu::RegisterAccess(wf).write_lane(dst_word, lane, accum[window] + sum);', + f' amdgpu::RegisterAccess(wf).write_lane(dst_word, lane, {value_expr});', ] ) L.append(' }') @@ -696,6 +710,7 @@ def gen_vector_mad_64_32( src: list[str], dtype: str | None, result_writer: str | None = None, + integer_clamp: bool = False, ) -> str: """Generate V_MAD_U64_U32 / V_MAD_I64_I32 body. @@ -723,16 +738,19 @@ def gen_vector_mad_64_32( L.append( f' uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64({src[2]}, lane);' ) - L.append( - ' uint64_t product = static_cast(s0) * static_cast(s1);' - ) + L.append(' uint64_t product = static_cast(s0 * s1);') L.append(' uint64_t result = product + s2;') + if writes_carry or integer_clamp: + L.append(' bool overflow = amdgpu::signed_add_overflows(product, s2);') if writes_carry: - L.append(' constexpr uint64_t sign_bit = 1ULL << 63;') + L.append(' if (overflow)') + L.append(' carry |= 1ULL << lane;') + if integer_clamp: + L.append(' if (inst_.clamp && overflow) {') L.append( - ' if (((~(product ^ s2) & (product ^ result)) & sign_bit) != 0)' + ' result = (product & (1ULL << 63)) ? std::bit_cast(std::numeric_limits::min()) : static_cast(std::numeric_limits::max());' ) - L.append(' carry |= 1ULL << lane;') + L.append(' }') L.append( f' amdgpu::RegisterAccess(wf).write_lane64({dst[0]}, lane, result);' ) @@ -748,9 +766,15 @@ def gen_vector_mad_64_32( ) L.append(' uint64_t product = s0 * s1;') L.append(' uint64_t result = product + s2;') + if writes_carry or integer_clamp: + L.append(' bool overflow = result < product;') if writes_carry: - L.append(' if (result < product)') + L.append(' if (overflow)') L.append(' carry |= 1ULL << lane;') + if integer_clamp: + L.append( + ' if (inst_.clamp && overflow) result = std::numeric_limits::max();' + ) L.append( f' amdgpu::RegisterAccess(wf).write_lane64({dst[0]}, lane, result);' ) @@ -803,8 +827,16 @@ def gen_vector_mad_32_16( L.append( f' int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane({src[2]}, lane));' ) + result = ( + 'amdgpu::vop3_integer_mad(' + 'static_cast(s0), static_cast(s1), ' + 'static_cast(s2), inst_.clamp)' + if is_vop3 + else 'static_cast(s0) * static_cast(s1) + ' + 'static_cast(s2)' + ) L.append( - f' amdgpu::RegisterAccess(wf).write_lane({dst[0]}, lane, static_cast(s0) * static_cast(s1) + static_cast(s2));' + f' amdgpu::RegisterAccess(wf).write_lane({dst[0]}, lane, {result});' ) else: if is_vop3: @@ -820,8 +852,13 @@ def gen_vector_mad_32_16( L.append( f' uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane({src[2]}, lane);' ) + result = ( + 'amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)' + if is_vop3 + else 's0 * s1 + s2' + ) L.append( - f' amdgpu::RegisterAccess(wf).write_lane({dst[0]}, lane, s0 * s1 + s2);' + f' amdgpu::RegisterAccess(wf).write_lane({dst[0]}, lane, {result});' ) L.append(' }') return '\n'.join(L) diff --git a/emulation/rocjitsu/lib/python/amdisa/isa_profile.py b/emulation/rocjitsu/lib/python/amdisa/isa_profile.py index bdc0e1502da9..d576ffe43674 100644 --- a/emulation/rocjitsu/lib/python/amdisa/isa_profile.py +++ b/emulation/rocjitsu/lib/python/amdisa/isa_profile.py @@ -580,6 +580,77 @@ def vop3p_source_modifier_omissions(self) -> frozenset[str]: """VOP3P instructions that do not use packed source modifier fields.""" return frozenset() + @property + def integer_clamp_dtypes(self) -> dict[str, str]: + """Integer instructions that require saturation-aware lowering. + + CLAMP is present in the shared VOP3 encoding, but it is not legal for + every opcode in that encoding. Keep the instruction-level policy here + instead of inferring support from the encoding field alone. ADD_MIN/MAX + always saturate their internal addition; the remaining instructions use + this policy to apply their encoded CLAMP modifier. + """ + return { + 'V_ADDC_CO_U32': 'u32', + 'V_ADD_MAX_I32': 'i32', + 'V_ADD_MAX_U32': 'u32', + 'V_ADD_MIN_I32': 'i32', + 'V_ADD_MIN_U32': 'u32', + 'V_ADD_CO_CI_U32': 'u32', + 'V_ADD_CO_U32': 'u32', + 'V_ADD_I16': 'i16', + 'V_ADD_I32': 'i32', + 'V_ADD_NC_I16': 'i16', + 'V_ADD_NC_I32': 'i32', + 'V_ADD_NC_U16': 'u16', + 'V_ADD_NC_U32': 'u32', + 'V_ADD_NC_U64': 'u64', + 'V_ADD_U16': 'u16', + 'V_ADD_U32': 'u32', + 'V_MAD_I16': 'i16', + 'V_MAD_I32_I16': 'i32', + 'V_MAD_I32_I24': 'i32', + 'V_MAD_I64_I32': 'i64', + 'V_MAD_CO_I64_I32': 'i64', + 'V_MAD_CO_U64_U32': 'u64', + 'V_MAD_NC_I64_I32': 'i64', + 'V_MAD_NC_U64_U32': 'u64', + 'V_MAD_U16': 'u16', + 'V_MAD_U32_U16': 'u32', + 'V_MAD_U32_U24': 'u32', + 'V_MAD_U64_U32': 'u64', + 'V_MQSAD_PK_U16_U8': 'u16', + 'V_MQSAD_U32_U8': 'u32', + 'V_MSAD_U8': 'u32', + 'V_MUL_I32_I24': 'i32', + 'V_MUL_U32_U24': 'u32', + 'V_PK_MAD_I16': 'i16', + 'V_PK_MAD_U16': 'u16', + 'V_SAD_HI_U8': 'u32', + 'V_SAD_U8': 'u32', + 'V_SAD_U16': 'u32', + 'V_SAD_U32': 'u32', + 'V_QSAD_PK_U16_U8': 'u16', + 'V_SUBBREV_CO_U32': 'u32', + 'V_SUBB_CO_U32': 'u32', + 'V_SUBREV_CO_CI_U32': 'u32', + 'V_SUBREV_CO_U32': 'u32', + 'V_SUBREV_NC_U32': 'u32', + 'V_SUBREV_U16': 'u16', + 'V_SUBREV_U32': 'u32', + 'V_SUB_CO_CI_U32': 'u32', + 'V_SUB_CO_U32': 'u32', + 'V_SUB_I16': 'i16', + 'V_SUB_I32': 'i32', + 'V_SUB_NC_I16': 'i16', + 'V_SUB_NC_I32': 'i32', + 'V_SUB_NC_U16': 'u16', + 'V_SUB_NC_U32': 'u32', + 'V_SUB_NC_U64': 'u64', + 'V_SUB_U16': 'u16', + 'V_SUB_U32': 'u32', + } + @property def dpp_ctrl_dialect(self) -> DppCtrlDialect: """DPP_CTRL naming and validity rules for this ISA.""" diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py index dd7e2a875bc9..d7e492e81222 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py @@ -1755,7 +1755,7 @@ def test_vop3_mad_u64_u32_writes_explicit_sdst_carry(): assert 'uint64_t carry = 0;' in body assert 'uint64_t product = s0 * s1;' in body - assert 'if (result < product)' in body + assert 'bool overflow = result < product;' in body assert 'carry |= 1ULL << lane;' in body assert 'amdgpu::write_wave_mask_scalar(sdst, wf, carry);' in body assert 'wf.set_vcc' not in body @@ -1770,6 +1770,35 @@ def test_vop3_mad_u64_u32_writes_explicit_sdst_carry(): assert 'write_wave_mask_scalar' not in callback_body +def test_vop3_mad_64_32_clamps_exact_result_without_changing_carry(): + unsigned = gen_vector_mad_64_32( + ['vdst', 'sdst'], + ['src0', 'src1', 'src2'], + 'u64', + integer_clamp=True, + ) + signed = gen_vector_mad_64_32( + ['vdst', 'sdst'], + ['src0', 'src1', 'src2'], + 'i64', + integer_clamp=True, + ) + + assert 'bool overflow = result < product;' in unsigned + assert 'inst_.clamp && overflow' in unsigned + assert unsigned.index('carry |= 1ULL << lane') < unsigned.index('inst_.clamp') + assert 'amdgpu::signed_add_overflows(product, s2)' in signed + assert 'inst_.clamp && overflow' in signed + assert '(product & (1ULL << 63))' in signed + assert signed.index('carry |= 1ULL << lane') < signed.index('inst_.clamp') + assert '__int128' not in unsigned + assert '__int128' not in signed + + policy = Cdna4Profile().integer_clamp_dtypes + assert policy['V_MAD_U64_U32'] == 'u64' + assert policy['V_MAD_I64_I32'] == 'i64' + + def test_vector_cmp_class_writes_explicit_sdst_mask(): body = gen_vector_cmp_class( ['sdst'], ['src0', 'src1'], 'f32', is_cmpx=False, is_vop3=True @@ -4077,6 +4106,135 @@ def test_gfx1250_generated_vop3_add_f16_applies_dpp( assert 'src0.clear_delegate();' not in body +def test_gfx1250_generator_wires_instruction_specific_integer_saturation(): + isa_xml = _mrisa_dir() / 'amdgpu_isa_cdna5.xml' + parser = Parser(str(isa_xml), Cdna5Profile()) + spec = parser.parse() + semantics = derive_all_semantics(spec) + generator = CodeGenerator(spec, '', semantics) + + def generated_body(name: str, enc_name: str, *, result_writer=None) -> str: + enc = next(enc for enc in spec.inst_encodings if enc.enc_name == enc_name) + inst = next(inst for inst in enc.insts if inst.name == name) + generator._current_inst_fields = {field.name for field in enc.ucode_fields} + generator._current_operand_names = {operand.name for operand in inst.operands} + generator._current_enc = enc + return generator._gen_execute_body( + inst, + semantics.instructions[name], + enc.enc_name, + result_writer=result_writer, + ) + + vop3 = generated_body('V_ADD_NC_U32', 'ENC_VOP3') + assert 'vop3_integer_add' in vop3 + assert 'inst_.clamp' in vop3 + + subrev = generated_body('V_SUBREV_NC_U32', 'ENC_VOP3') + assert 'vop3_integer_sub' in subrev + assert subrev.index('read_lane(src1, lane)') < subrev.index('read_lane(src0, lane)') + assert 'inst_.clamp' in subrev + + add_u64 = generated_body('V_ADD_NC_U64', 'ENC_VOP3') + assert 'vop3_integer_add' in add_u64 + assert 'inst_.clamp' in add_u64 + + sub_u64 = generated_body('V_SUB_NC_U64', 'ENC_VOP3') + assert 'vop3_integer_sub' in sub_u64 + assert 'inst_.clamp' in sub_u64 + + expected_helpers = { + 'V_ADD_MAX_I32': 'vop3_integer_add_minmax', + 'V_ADD_MAX_U32': 'vop3_integer_add_minmax', + 'V_ADD_MIN_I32': 'vop3_integer_add_minmax', + 'V_ADD_MIN_U32': 'vop3_integer_add_minmax', + } + for name, helper in expected_helpers.items(): + body = generated_body(name, 'ENC_VOP3') + assert helper in body + assert 'inst_.clamp' not in body + + for name, enc_name, result_writer in ( + ('V_MAD_NC_U64_U32', 'ENC_VOP3', None), + ('V_MAD_NC_I64_I32', 'ENC_VOP3', None), + ('V_MAD_CO_U64_U32', 'VOP3_SDST_ENC', 'commit_result'), + ('V_MAD_CO_I64_I32', 'VOP3_SDST_ENC', 'commit_result'), + ): + body = generated_body(name, enc_name, result_writer=result_writer) + assert 'bool overflow' in body + assert '__int128' not in body + assert 'inst_.clamp' in body + + for name, helper in ( + ('V_PK_MAD_I16', 'vop3_integer_mad'), + ('V_PK_MAD_U16', 'vop3_integer_mad'), + ): + body = generated_body(name, 'ENC_VOP3P') + assert body.count(helper) == 2 + assert body.count('inst_.clamp') == 2 + + for name in ('V_QSAD_PK_U16_U8', 'V_MQSAD_PK_U16_U8', 'V_MQSAD_U32_U8'): + body = generated_body(name, 'ENC_VOP3') + assert 'inst_.clamp' in body + + vop2 = generated_body('V_ADD_NC_U32', 'ENC_VOP2') + assert 'vop3_integer_add' not in vop2 + assert 'inst_.clamp' not in vop2 + + carry = generated_body( + 'V_ADD_CO_U32', 'VOP3_SDST_ENC', result_writer='commit_result' + ) + assert 'inst_.clamp && w > 0xFFFFFFFFULL ? UINT32_MAX' in carry + assert 'inst_.clamp' in carry + assert 'if (w > 0xFFFFFFFFULL) vcc |=' in carry + + +def test_rdna4_generator_uses_instruction_policy_for_integer_clamp(): + isa_xml = _mrisa_dir() / 'amdgpu_isa_rdna4.xml' + parser = Parser(str(isa_xml), Rdna4Profile()) + spec = parser.parse() + semantics = derive_all_semantics(spec) + generator = CodeGenerator(spec, '', semantics) + enc = next(enc for enc in spec.inst_encodings if enc.enc_name == 'ENC_VOP3') + generator._current_inst_fields = {field.name for field in enc.ucode_fields} + generator._current_enc = enc + + def generated_body(name: str) -> str: + inst = next(inst for inst in enc.insts if inst.name == name) + generator._current_operand_names = {operand.name for operand in inst.operands} + return generator._gen_execute_body( + inst, semantics.instructions[name], enc.enc_name + ) + + add3 = generated_body('V_ADD3_U32') + assert 'vop3_integer_' not in add3 + assert 'inst_.clamp' not in add3 + + expected_helpers = { + 'V_MUL_I32_I24': 'vop3_integer_mul', + 'V_MUL_U32_U24': 'vop3_integer_mul', + 'V_MAD_I32_I24': 'vop3_integer_mad', + 'V_MAD_U32_U24': 'vop3_integer_mad', + 'V_MAD_I16': 'vop3_integer_mad', + 'V_MAD_U16': 'vop3_integer_mad', + 'V_MAD_I32_I16': 'vop3_integer_mad', + 'V_MAD_U32_U16': 'vop3_integer_mad', + 'V_SAD_HI_U8': 'vop3_integer_sad_hi_u8', + 'V_SAD_U8': 'vop3_integer_sad_u8', + 'V_SAD_U16': 'vop3_integer_sad_u16', + 'V_SAD_U32': 'vop3_integer_sad_u32', + 'V_MSAD_U8': 'vop3_integer_msad_u8', + } + for name, helper in expected_helpers.items(): + body = generated_body(name) + assert helper in body + assert 'inst_.clamp' in body + + or3 = generated_body('V_OR3_B32') + assert 'vop3_integer_add3' not in or3 + assert 'inst_.clamp' not in or3 + + def test_noop_format_validation_is_inherited(amdgpu_generated_root: Path): encodings_h = (amdgpu_generated_root / 'rdna4' / 'encodings.h').read_text() diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py index 97b094478b18..8ffe441572d2 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py @@ -121,6 +121,32 @@ def test_pk_add_minmax_saturates_add_before_selecting_third_operand(): assert 'std::min(sum_lo, c_lo)' in unsigned +def test_pk_mad_integer_clamp_saturates_both_selected_halves(): + signed = gen_pk_ternary( + ['vdst'], + ['src0', 'src1', 'src2'], + 'mad', + 'i16', + op_sel_hi_2_expr='inst_.op_sel_hi_2', + opsel_exprs=('inst_.op_sel', 'inst_.op_sel_hi'), + integer_clamp=True, + ) + unsigned = gen_pk_ternary( + ['vdst'], + ['src0', 'src1', 'src2'], + 'mad', + 'u16', + op_sel_hi_2_expr='inst_.op_sel_hi_2', + opsel_exprs=('inst_.op_sel', 'inst_.op_sel_hi'), + integer_clamp=True, + ) + + assert signed.count('vop3_integer_mad') == 2 + assert signed.count('inst_.clamp') == 2 + assert unsigned.count('vop3_integer_mad') == 2 + assert unsigned.count('inst_.clamp') == 2 + + def test_dot8_iu4_uses_operand_signedness_modifiers(): cpp = gen_dot8(['vdst'], ['src0', 'src1', 'src2'], 'dot8_i32_iu4') diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py index e963a575c0b5..fc616ac1ebd2 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py @@ -1579,6 +1579,56 @@ def test_signed_add_sub_use_unsigned_operands_to_avoid_ub(self): not in cpp ) + @pytest.mark.parametrize( + 'op,dtype,expected', + [ + ('add', 'u32', 'vop3_integer_add'), + ('sub', 'u32', 'vop3_integer_sub'), + ('subrev', 'u32', 'vop3_integer_sub'), + ('add', 'i32', 'vop3_integer_add'), + ('sub', 'i32', 'vop3_integer_sub'), + ('add', 'u16', 'vop3_integer_add'), + ('sub', 'i16', 'vop3_integer_sub'), + ('add', 'u64', 'vop3_integer_add'), + ('sub', 'u64', 'vop3_integer_sub'), + ], + ) + def test_vop3_integer_clamp_saturates_before_narrowing(self, op, dtype, expected): + sem = _FakeSem(f'V_{op.upper()}_{dtype.upper()}', 'vector_binop', op, dtype) + block = derive_sema_block(sem) + ctx = LoweringContext( + exec_model=block.pragma, + integer_saturation_dtype=dtype, + ) + + cpp = lower_sema_block(block, ctx) + + assert 'inst_.clamp' in cpp + assert expected in cpp + + @pytest.mark.parametrize( + 'name,op,dtype,helper', + [ + ('V_MUL_I32_I24', 'mul', 'i24', 'vop3_integer_mul'), + ('V_MUL_U32_U24', 'mul', 'u24', 'vop3_integer_mul'), + ('V_MAD_I32_I24', 'mad', 'i24', 'vop3_integer_mad'), + ('V_MAD_U32_U24', 'mad', 'u24', 'vop3_integer_mad'), + ], + ) + def test_vop3_integer_mul_mad_use_exact_saturating_intermediate( + self, name, op, dtype, helper + ): + sem_class = 'vector_binop' if op == 'mul' else 'vector_ternary' + sem = _FakeSem(name, sem_class, op, dtype) + block = derive_sema_block(sem) + cpp = lower_sema_block( + block, + LoweringContext(exec_model=block.pragma, integer_saturation_dtype=dtype), + ) + + assert helper in cpp + assert 'inst_.clamp' in cpp + def test_lshlrev(self): sem = _FakeSem('V_LSHLREV_B32', 'vector_binop', 'lshlrev', 'b32') block = derive_sema_block(sem) @@ -1655,7 +1705,7 @@ def test_lowers_to_std_fma(self): cpp = lower_sema_block(block) assert 'std::fma(' in cpp - def test_add_minmax_i32_u32_wraps_add_before_clamp(self): + def test_add_minmax_i32_u32_intrinsically_saturates_add_before_selection(self): cases = [ ('V_ADD_MAX_I32', 'add_max', 'i32', 'std::max'), ('V_ADD_MAX_U32', 'add_max', 'u32', 'std::max'), @@ -1670,10 +1720,17 @@ def test_add_minmax_i32_u32_wraps_add_before_clamp(self): assert sem.data_type == dtype block = derive_sema_block(sem) - cpp = lower_sema_block(block) - assert clamp_fn in cpp - assert 'static_cast' in cpp - assert f'{op}_{dtype}(' not in cpp + cpp = lower_sema_block( + block, + LoweringContext( + exec_model=ExecModel.VECTOR, + integer_saturation_dtype=dtype, + ), + ) + select_max = 'true' if clamp_fn == 'std::max' else 'false' + assert f'vop3_integer_add_minmax<' in cpp + assert f', {select_max}>' in cpp + assert 'inst_.clamp' not in cpp def test_ashr_pk_i8_u8_i32_packs_shifted_saturated_bytes(self): cases = [ @@ -1853,6 +1910,33 @@ def test_sub_co(self): } assert 'sub_co' in call_names + @pytest.mark.parametrize( + ('operation', 'expected'), + [ + ('add', 'inst_.clamp && w > 0xFFFFFFFFULL ? UINT32_MAX'), + ('sub', 'inst_.clamp && a < b ? 0u'), + ('rsub', 'inst_.clamp && a < b ? 0u'), + ('addc', 'inst_.clamp && w > 0xFFFFFFFFULL ? UINT32_MAX'), + ('subbc', 'inst_.clamp && a < b ? 0u'), + ('subbrevco', 'inst_.clamp && a < b ? 0u'), + ], + ) + def test_vop3_clamp_saturates_result_without_changing_carry( + self, operation, expected + ): + sem = _FakeSem('V_CARRY_U32', 'vector_add_co', operation, 'u32') + block = derive_sema_block(sem) + cpp = lower_sema_block( + block, + LoweringContext( + exec_model=block.pragma, + integer_saturation_dtype='u32', + ), + ) + + assert expected in cpp + assert 'vcc |= (1ULL << lane)' in cpp + class TestDeriveVectorCndmask: def test_cndmask(self): @@ -2426,6 +2510,29 @@ def test_pk_binop_bf16_generator_uses_bf16_helpers(self): assert 'util::f32_to_bf16' in cpp assert 'util::f32_to_f16' not in cpp + @pytest.mark.parametrize( + ('operation', 'data_type', 'helper'), + [ + ('add', 'i16', 'vop3_integer_add'), + ('sub', 'i16', 'vop3_integer_sub'), + ('add', 'u16', 'vop3_integer_add'), + ('sub', 'u16', 'vop3_integer_sub'), + ], + ) + def test_pk_integer_add_sub_clamps_each_selected_half( + self, operation, data_type, helper + ): + cpp = gen_pk_binop( + ['vdst'], + ['src0', 'src1'], + operation, + data_type, + ('inst_.opsel', 'inst_.opsel_hi'), + ) + + assert cpp.count(helper) == 2 + assert cpp.count('inst_.clamp') == 2 + def test_pk_ternary_bf16_generator_uses_fma_and_bf16_helpers(self): cpp = gen_pk_ternary( ['vdst'], diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_vector_special_codegen.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_vector_special_codegen.py index b5f242b82a34..236b26aaab04 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_vector_special_codegen.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_vector_special_codegen.py @@ -112,8 +112,8 @@ def test_saturating_pack_codegen_covers_all_narrowing_modes(): def test_perm_and_qsad_codegen_cover_multiword_results(): perm = gen_vector_perm_pk16(['vdst'], ['src0', 'src1', 'src2'], 'b6', True) - qsad = gen_vector_qsad(['vdst'], ['src0', 'src1', 'src2'], 'sad_pk_u16', True) - mqsad = gen_vector_qsad(['vdst'], ['src0', 'src1', 'src2'], 'msad_u32', True) + qsad = gen_vector_qsad(['vdst'], ['src0', 'src1', 'src2'], 'sad_pk_u16', True, True) + mqsad = gen_vector_qsad(['vdst'], ['src0', 'src1', 'src2'], 'msad_u32', True, True) assert 'uint32_t packed[3]' in perm assert 'source_bit = index * 6u' in perm @@ -122,11 +122,11 @@ def test_perm_and_qsad_codegen_cover_multiword_results(): assert 'read_lane64(src1, lane)' in perm assert '(window + byte) * 8' in qsad assert 'write_lane64(vdst, lane, packed_result)' in qsad - assert 'value & 0xffffu' in qsad + assert 'inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)' in qsad assert 'if (b != 0)' in mqsad assert 'accum[window] = amdgpu::RegisterAccess(wf).read_lane' in mqsad assert 'if (std::optional constant = src2.const_value())' in mqsad - assert 'accum[window] + sum' in mqsad + assert 'vop3_integer_add(accum[window], sum, inst_.clamp)' in mqsad @pytest.mark.parametrize( diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3_exec.cpp index 2e1d1987ea7b..98fe898af04f 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/vop3_exec.cpp @@ -1438,11 +1438,13 @@ void VAddU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1455,11 +1457,13 @@ void VSubU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1472,11 +1476,13 @@ void VSubrevU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1989,7 +1995,9 @@ void VQsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -2013,7 +2021,9 @@ void VMqsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -2051,7 +2061,9 @@ void VMqsadU32U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } dst_word.encoding_value_ = vdst.encoding_value_ + static_cast(window); - amdgpu::sdwa::write_lane(*this, wf, dst_word, lane, accum[window] + sum); + amdgpu::sdwa::write_lane( + *this, wf, dst_word, lane, + amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)); } } } @@ -2273,7 +2285,8 @@ void VMadU32U16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } } @@ -2294,9 +2307,10 @@ void VMadI32I16Vop3::execute_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } } @@ -2787,7 +2801,7 @@ void VMadU16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -2801,10 +2815,7 @@ void VMadI16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -3034,10 +3045,10 @@ void VAddI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -3050,10 +3061,10 @@ void VSubI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3_exec.cpp index b7ad72ed9d1e..160c2c351b3d 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/vop3_exec.cpp @@ -1430,11 +1430,13 @@ void VAddU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1447,11 +1449,13 @@ void VSubU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1464,11 +1468,13 @@ void VSubrevU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1981,7 +1987,9 @@ void VQsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -2005,7 +2013,9 @@ void VMqsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -2043,7 +2053,9 @@ void VMqsadU32U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } dst_word.encoding_value_ = vdst.encoding_value_ + static_cast(window); - amdgpu::sdwa::write_lane(*this, wf, dst_word, lane, accum[window] + sum); + amdgpu::sdwa::write_lane( + *this, wf, dst_word, lane, + amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)); } } } @@ -2265,7 +2277,8 @@ void VMadU32U16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } } @@ -2286,9 +2299,10 @@ void VMadI32I16Vop3::execute_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } } @@ -2779,7 +2793,7 @@ void VMadU16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -2793,10 +2807,7 @@ void VMadI16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -3006,10 +3017,10 @@ void VAddI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -3022,10 +3033,10 @@ void VSubI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3_exec.cpp index be274c1aab61..fe330f7b857d 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/vop3_exec.cpp @@ -1505,11 +1505,13 @@ void VAddU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1522,11 +1524,13 @@ void VSubU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1539,11 +1543,13 @@ void VSubrevU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -2050,7 +2056,9 @@ void VQsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -2074,7 +2082,9 @@ void VMqsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -2112,7 +2122,9 @@ void VMqsadU32U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } dst_word.encoding_value_ = vdst.encoding_value_ + static_cast(window); - amdgpu::sdwa::write_lane(*this, wf, dst_word, lane, accum[window] + sum); + amdgpu::sdwa::write_lane( + *this, wf, dst_word, lane, + amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)); } } } @@ -2334,7 +2346,8 @@ void VMadU32U16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } } @@ -2355,9 +2368,10 @@ void VMadI32I16Vop3::execute_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } } @@ -2848,7 +2862,7 @@ void VMadU16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -2862,10 +2876,7 @@ void VMadI16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -3079,10 +3090,10 @@ void VAddI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -3095,10 +3106,10 @@ void VSubI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3_exec.cpp index 97a6b5ac5ad7..527d37f95a4b 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/vop3_exec.cpp @@ -1565,11 +1565,13 @@ void VAddU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1582,11 +1584,13 @@ void VSubU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -1599,11 +1603,13 @@ void VSubrevU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -2110,7 +2116,9 @@ void VQsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -2134,7 +2142,9 @@ void VMqsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -2172,7 +2182,9 @@ void VMqsadU32U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } dst_word.encoding_value_ = vdst.encoding_value_ + static_cast(window); - amdgpu::sdwa::write_lane(*this, wf, dst_word, lane, accum[window] + sum); + amdgpu::sdwa::write_lane( + *this, wf, dst_word, lane, + amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)); } } } @@ -2394,7 +2406,8 @@ void VMadU32U16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } } @@ -2415,9 +2428,10 @@ void VMadI32I16Vop3::execute_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } } @@ -2908,7 +2922,7 @@ void VMadU16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -2922,10 +2936,7 @@ void VMadI16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -7450,10 +7461,10 @@ void VAddI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -7466,10 +7477,10 @@ void VSubI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_alu.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_alu.cpp index 8f510f3e4c39..543c7867c905 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_alu.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_alu.cpp @@ -4033,8 +4033,10 @@ void VAddNcU64Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane64( *this, wf, vdst, lane, - (static_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)) + - static_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)))); + amdgpu::vop3_integer_add( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)), + inst_.clamp)); } } @@ -4058,8 +4060,10 @@ RJ_NOINLINE void VAddNcU64Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane64( *this, wf, vdst, lane, - (static_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)) + - static_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)))); + amdgpu::vop3_integer_add( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)), + inst_.clamp)); } dpp_write_mask_scope_.restore(); } @@ -4075,8 +4079,10 @@ void VSubNcU64Vop3::execute_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane64( *this, wf, vdst, lane, - (static_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)) - - static_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)))); + amdgpu::vop3_integer_sub( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)), + inst_.clamp)); } } @@ -4100,8 +4106,10 @@ RJ_NOINLINE void VSubNcU64Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { continue; amdgpu::sdwa::write_lane64( *this, wf, vdst, lane, - (static_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)) - - static_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)))); + amdgpu::vop3_integer_sub( + static_cast(amdgpu::RegisterAccess(wf).read_lane64(src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane64(src1, lane)), + inst_.clamp)); } dpp_write_mask_scope_.restore(); } @@ -5718,11 +5726,13 @@ void VAddNcU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -5752,11 +5762,13 @@ RJ_NOINLINE void VAddNcU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -5774,11 +5786,13 @@ void VSubNcU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -5808,11 +5822,13 @@ RJ_NOINLINE void VSubNcU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -6120,10 +6136,10 @@ void VAddNcI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -6153,10 +6169,10 @@ RJ_NOINLINE void VAddNcI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -6174,10 +6190,10 @@ void VSubNcI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -6207,10 +6223,10 @@ RJ_NOINLINE void VSubNcI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_misc.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_misc.cpp index 4deec9eca8b6..a5f90f58d1d6 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_misc.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_misc.cpp @@ -247,7 +247,9 @@ void VQsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -271,7 +273,9 @@ void VMqsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -309,7 +313,9 @@ void VMqsadU32U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } dst_word.encoding_value_ = vdst.encoding_value_ + static_cast(window); - amdgpu::sdwa::write_lane(*this, wf, dst_word, lane, accum[window] + sum); + amdgpu::sdwa::write_lane( + *this, wf, dst_word, lane, + amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)); } } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp index e1f4e8db228f..03066b01a3df 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3_exec_ternary.cpp @@ -1760,7 +1760,7 @@ void VMadU16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -1791,7 +1791,7 @@ RJ_NOINLINE void VMadU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -2478,10 +2478,7 @@ void VMadI16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -2512,10 +2509,7 @@ RJ_NOINLINE void VMadI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -2650,7 +2644,8 @@ void VMadU32U16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } } @@ -2683,7 +2678,8 @@ RJ_NOINLINE void VMadU32U16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } dpp_write_mask_scope_.restore(); } @@ -2709,9 +2705,10 @@ void VMadI32I16Vop3::execute_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } } @@ -2749,9 +2746,10 @@ RJ_NOINLINE void VMadI32I16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } dpp_write_mask_scope_.restore(); } @@ -2765,17 +2763,12 @@ void VAddMaxI32Vop3::execute_impl(amdgpu::Wavefront &wf) { for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, [&]() { - uint32_t sum_bits = - static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))) + - static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane))); - int32_t sum = static_cast(sum_bits); - int32_t clamp = static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane))); - return static_cast(std::max(sum, clamp)); - }()); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_add_minmax( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)))); } } @@ -2799,17 +2792,12 @@ RJ_NOINLINE void VAddMaxI32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, [&]() { - uint32_t sum_bits = - static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))) + - static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane))); - int32_t sum = static_cast(sum_bits); - int32_t clamp = static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane))); - return static_cast(std::max(sum, clamp)); - }()); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_add_minmax( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)))); } dpp_write_mask_scope_.restore(); } @@ -2823,11 +2811,11 @@ void VAddMaxU32Vop3::execute_impl(amdgpu::Wavefront &wf) { for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - amdgpu::sdwa::write_lane( - *this, wf, vdst, lane, - std::max(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)) + - static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)), - static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)))); + amdgpu::sdwa::write_lane(*this, wf, vdst, lane, + amdgpu::vop3_integer_add_minmax( + amdgpu::RegisterAccess(wf).read_lane(src0, lane), + amdgpu::RegisterAccess(wf).read_lane(src1, lane), + amdgpu::RegisterAccess(wf).read_lane(src2, lane))); } } @@ -2851,11 +2839,11 @@ RJ_NOINLINE void VAddMaxU32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - amdgpu::sdwa::write_lane( - *this, wf, vdst, lane, - std::max(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)) + - static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)), - static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)))); + amdgpu::sdwa::write_lane(*this, wf, vdst, lane, + amdgpu::vop3_integer_add_minmax( + amdgpu::RegisterAccess(wf).read_lane(src0, lane), + amdgpu::RegisterAccess(wf).read_lane(src1, lane), + amdgpu::RegisterAccess(wf).read_lane(src2, lane))); } dpp_write_mask_scope_.restore(); } @@ -2869,17 +2857,12 @@ void VAddMinI32Vop3::execute_impl(amdgpu::Wavefront &wf) { for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, [&]() { - uint32_t sum_bits = - static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))) + - static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane))); - int32_t sum = static_cast(sum_bits); - int32_t clamp = static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane))); - return static_cast(std::min(sum, clamp)); - }()); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_add_minmax( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)))); } } @@ -2903,17 +2886,12 @@ RJ_NOINLINE void VAddMinI32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, [&]() { - uint32_t sum_bits = - static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane))) + - static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane))); - int32_t sum = static_cast(sum_bits); - int32_t clamp = static_cast( - static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane))); - return static_cast(std::min(sum, clamp)); - }()); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_add_minmax( + static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)))); } dpp_write_mask_scope_.restore(); } @@ -2927,11 +2905,11 @@ void VAddMinU32Vop3::execute_impl(amdgpu::Wavefront &wf) { for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - amdgpu::sdwa::write_lane( - *this, wf, vdst, lane, - std::min(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)) + - static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)), - static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)))); + amdgpu::sdwa::write_lane(*this, wf, vdst, lane, + amdgpu::vop3_integer_add_minmax( + amdgpu::RegisterAccess(wf).read_lane(src0, lane), + amdgpu::RegisterAccess(wf).read_lane(src1, lane), + amdgpu::RegisterAccess(wf).read_lane(src2, lane))); } } @@ -2955,11 +2933,11 @@ RJ_NOINLINE void VAddMinU32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - amdgpu::sdwa::write_lane( - *this, wf, vdst, lane, - std::min(static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)) + - static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)), - static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)))); + amdgpu::sdwa::write_lane(*this, wf, vdst, lane, + amdgpu::vop3_integer_add_minmax( + amdgpu::RegisterAccess(wf).read_lane(src0, lane), + amdgpu::RegisterAccess(wf).read_lane(src1, lane), + amdgpu::RegisterAccess(wf).read_lane(src2, lane))); } dpp_write_mask_scope_.restore(); } @@ -3996,6 +3974,9 @@ void VMadNcU64U32Vop3::execute_impl(amdgpu::Wavefront &wf) { uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(src2, lane); uint64_t product = s0 * s1; uint64_t result = product + s2; + bool overflow = result < product; + if (inst_.clamp && overflow) + result = std::numeric_limits::max(); amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, result); } } @@ -4023,6 +4004,9 @@ RJ_NOINLINE void VMadNcU64U32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(src2, lane); uint64_t product = s0 * s1; uint64_t result = product + s2; + bool overflow = result < product; + if (inst_.clamp && overflow) + result = std::numeric_limits::max(); amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, result); } dpp_write_mask_scope_.restore(); @@ -4040,8 +4024,14 @@ void VMadNcI64I32Vop3::execute_impl(amdgpu::Wavefront &wf) { int64_t s0 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); int64_t s1 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(src2, lane); - uint64_t product = static_cast(s0) * static_cast(s1); + uint64_t product = static_cast(s0 * s1); uint64_t result = product + s2; + bool overflow = amdgpu::signed_add_overflows(product, s2); + if (inst_.clamp && overflow) { + result = (product & (1ULL << 63)) + ? std::bit_cast(std::numeric_limits::min()) + : static_cast(std::numeric_limits::max()); + } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, result); } } @@ -4067,8 +4057,14 @@ RJ_NOINLINE void VMadNcI64I32Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) int64_t s0 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src0, lane)); int64_t s1 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src1, lane)); uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(src2, lane); - uint64_t product = static_cast(s0) * static_cast(s1); + uint64_t product = static_cast(s0 * s1); uint64_t result = product + s2; + bool overflow = amdgpu::signed_add_overflows(product, s2); + if (inst_.clamp && overflow) { + result = (product & (1ULL << 63)) + ? std::bit_cast(std::numeric_limits::min()) + : static_cast(std::numeric_limits::max()); + } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, result); } dpp_write_mask_scope_.restore(); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp index 189e3ba345c4..e62628f702a7 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp @@ -159,8 +159,12 @@ void VPkMadI16Vop3p::execute_impl(amdgpu::Wavefront &wf) { int16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); int16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); int16_t c_hi = static_cast(sel2_hi ? (raw2 >> 16) : raw2); - uint16_t rlo = static_cast(static_cast(a_lo) * b_lo + c_lo); - uint16_t rhi = static_cast(static_cast(a_hi) * b_hi + c_hi); + uint16_t rlo = amdgpu::vop3_integer_mad(static_cast(a_lo), + static_cast(b_lo), + static_cast(c_lo), inst_.clamp); + uint16_t rhi = amdgpu::vop3_integer_mad(static_cast(a_hi), + static_cast(b_hi), + static_cast(c_hi), inst_.clamp); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -203,8 +207,10 @@ void VPkAddI16Vop3p::execute_impl(amdgpu::Wavefront &wf) { int16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); int16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); int16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo + b_lo); - uint16_t rhi = static_cast(a_hi + b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_add(a_lo, b_lo, inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_add(a_hi, b_hi, inst_.clamp)); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -225,8 +231,10 @@ void VPkSubI16Vop3p::execute_impl(amdgpu::Wavefront &wf) { int16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); int16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); int16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo - b_lo); - uint16_t rhi = static_cast(a_hi - b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_sub(a_lo, b_lo, inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_sub(a_hi, b_hi, inst_.clamp)); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -362,8 +370,8 @@ void VPkMadU16Vop3p::execute_impl(amdgpu::Wavefront &wf) { uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); uint16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); uint16_t c_hi = static_cast(sel2_hi ? (raw2 >> 16) : raw2); - uint16_t rlo = static_cast(static_cast(a_lo) * b_lo + c_lo); - uint16_t rhi = static_cast(static_cast(a_hi) * b_hi + c_hi); + uint16_t rlo = amdgpu::vop3_integer_mad(a_lo, b_lo, c_lo, inst_.clamp); + uint16_t rhi = amdgpu::vop3_integer_mad(a_hi, b_hi, c_hi, inst_.clamp); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -384,8 +392,10 @@ void VPkAddU16Vop3p::execute_impl(amdgpu::Wavefront &wf) { uint16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); uint16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo + b_lo); - uint16_t rhi = static_cast(a_hi + b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_add(a_lo, b_lo, inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_add(a_hi, b_hi, inst_.clamp)); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -406,8 +416,10 @@ void VPkSubU16Vop3p::execute_impl(amdgpu::Wavefront &wf) { uint16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); uint16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo - b_lo); - uint16_t rhi = static_cast(a_hi - b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_sub(a_lo, b_lo, inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_sub(a_hi, b_hi, inst_.clamp)); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vop3_exec.cpp index 6cd9f37de85a..961623400c62 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/vop3_exec.cpp @@ -645,7 +645,9 @@ void VQsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -669,7 +671,9 @@ void VMqsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -711,7 +715,9 @@ void VMqsadU32U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } dst_word.encoding_value_ = vdst.encoding_value_ + static_cast(window); - amdgpu::sdwa::write_lane(*this, wf, dst_word, lane, accum[window] + sum); + amdgpu::sdwa::write_lane( + *this, wf, dst_word, lane, + amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)); } } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vop3_exec.cpp index 5df3ac53f9cf..ea6b2fbb05b3 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/vop3_exec.cpp @@ -687,7 +687,9 @@ void VQsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -711,7 +713,9 @@ void VMqsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -753,7 +757,9 @@ void VMqsadU32U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } dst_word.encoding_value_ = vdst.encoding_value_ + static_cast(window); - amdgpu::sdwa::write_lane(*this, wf, dst_word, lane, accum[window] + sum); + amdgpu::sdwa::write_lane( + *this, wf, dst_word, lane, + amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)); } } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop3_exec.cpp index a0575d034104..5efbefe76445 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/vop3_exec.cpp @@ -6612,7 +6612,9 @@ void VQsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -6636,7 +6638,9 @@ void VMqsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -6674,7 +6678,9 @@ void VMqsadU32U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } dst_word.encoding_value_ = vdst.encoding_value_ + static_cast(window); - amdgpu::sdwa::write_lane(*this, wf, dst_word, lane, accum[window] + sum); + amdgpu::sdwa::write_lane( + *this, wf, dst_word, lane, + amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)); } } } @@ -6720,7 +6726,7 @@ void VMadU16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -6751,7 +6757,7 @@ RJ_NOINLINE void VMadU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -7926,10 +7932,7 @@ void VMadI16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -7960,10 +7963,7 @@ RJ_NOINLINE void VMadI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -8258,7 +8258,8 @@ void VMadU32U16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } } @@ -8291,7 +8292,8 @@ RJ_NOINLINE void VMadU32U16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } dpp_write_mask_scope_.restore(); } @@ -8317,9 +8319,10 @@ void VMadI32I16Vop3::execute_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } } @@ -8357,9 +8360,10 @@ RJ_NOINLINE void VMadI32I16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } dpp_write_mask_scope_.restore(); } @@ -9199,11 +9203,13 @@ void VAddNcU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9233,11 +9239,13 @@ RJ_NOINLINE void VAddNcU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9255,11 +9263,13 @@ void VSubNcU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9289,11 +9299,13 @@ RJ_NOINLINE void VSubNcU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9701,10 +9713,10 @@ void VAddNcI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9734,10 +9746,10 @@ RJ_NOINLINE void VAddNcI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9755,10 +9767,10 @@ void VSubNcI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9788,10 +9800,10 @@ RJ_NOINLINE void VSubNcI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop3_exec.cpp index c7efd85fc66f..45b3ab8c8c18 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/vop3_exec.cpp @@ -6612,7 +6612,9 @@ void VQsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -6636,7 +6638,9 @@ void VMqsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -6674,7 +6678,9 @@ void VMqsadU32U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } dst_word.encoding_value_ = vdst.encoding_value_ + static_cast(window); - amdgpu::sdwa::write_lane(*this, wf, dst_word, lane, accum[window] + sum); + amdgpu::sdwa::write_lane( + *this, wf, dst_word, lane, + amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)); } } } @@ -6720,7 +6726,7 @@ void VMadU16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -6751,7 +6757,7 @@ RJ_NOINLINE void VMadU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -7926,10 +7932,7 @@ void VMadI16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -7960,10 +7963,7 @@ RJ_NOINLINE void VMadI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -8258,7 +8258,8 @@ void VMadU32U16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } } @@ -8291,7 +8292,8 @@ RJ_NOINLINE void VMadU32U16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } dpp_write_mask_scope_.restore(); } @@ -8317,9 +8319,10 @@ void VMadI32I16Vop3::execute_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } } @@ -8357,9 +8360,10 @@ RJ_NOINLINE void VMadI32I16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } dpp_write_mask_scope_.restore(); } @@ -9199,11 +9203,13 @@ void VAddNcU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9233,11 +9239,13 @@ RJ_NOINLINE void VAddNcU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9255,11 +9263,13 @@ void VSubNcU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9289,11 +9299,13 @@ RJ_NOINLINE void VSubNcU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9701,10 +9713,10 @@ void VAddNcI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9734,10 +9746,10 @@ RJ_NOINLINE void VAddNcI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9755,10 +9767,10 @@ void VSubNcI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -9788,10 +9800,10 @@ RJ_NOINLINE void VSubNcI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3_exec.cpp index 1dff41bf50f6..ccef57f1c9b0 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3_exec.cpp @@ -7649,7 +7649,9 @@ void VQsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -7673,7 +7675,9 @@ void VMqsadPkU16U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } uint32_t value = sum + ((accum >> (window * 16)) & 0xffffu); - packed_result |= static_cast(value & 0xffffu) << (window * 16); + packed_result |= + static_cast(inst_.clamp ? std::min(value, 0xffffu) : (value & 0xffffu)) + << (window * 16); } amdgpu::sdwa::write_lane64(*this, wf, vdst, lane, packed_result); } @@ -7711,7 +7715,9 @@ void VMqsadU32U8Vop3::execute_impl(amdgpu::Wavefront &wf) { sum += a > b ? a - b : b - a; } dst_word.encoding_value_ = vdst.encoding_value_ + static_cast(window); - amdgpu::sdwa::write_lane(*this, wf, dst_word, lane, accum[window] + sum); + amdgpu::sdwa::write_lane( + *this, wf, dst_word, lane, + amdgpu::vop3_integer_add(accum[window], sum, inst_.clamp)); } } } @@ -7757,7 +7763,7 @@ void VMadU16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -7788,7 +7794,7 @@ RJ_NOINLINE void VMadU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - uint32_t result = (s0 * s1 + s2) & 0xffffu; + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -8477,10 +8483,7 @@ void VMadI16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } } @@ -8511,10 +8514,7 @@ RJ_NOINLINE void VMadI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = ::rocjitsu::amdgpu::read_vop3_true16_src(src2, wf, lane, opsel, 2); - int32_t a = static_cast(s0); - int32_t b = static_cast(s1); - int32_t c = static_cast(s2); - uint32_t result = static_cast(static_cast(a * b + c)); + uint32_t result = amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, result, true); } @@ -8809,7 +8809,8 @@ void VMadU32U16Vop3::execute_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } } @@ -8842,7 +8843,8 @@ RJ_NOINLINE void VMadU32U16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(src2, lane); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, s0 * s1 + s2); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, amdgpu::vop3_integer_mad(s0, s1, s2, inst_.clamp)); } dpp_write_mask_scope_.restore(); } @@ -8868,9 +8870,10 @@ void VMadI32I16Vop3::execute_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } } @@ -8908,9 +8911,10 @@ RJ_NOINLINE void VMadI32I16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(src2, lane)); - amdgpu::sdwa::write_lane(*this, wf, vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + amdgpu::sdwa::write_lane( + *this, wf, vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst_.clamp)); } dpp_write_mask_scope_.restore(); } @@ -10212,11 +10216,13 @@ void VAddNcU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -10246,11 +10252,13 @@ RJ_NOINLINE void VAddNcU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) + - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -10268,11 +10276,13 @@ void VSubNcU16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -10302,11 +10312,13 @@ RJ_NOINLINE void VSubNcU16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast(static_cast(static_cast( - static_cast((static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src0, wf, lane, opsel, 0)) - - static_cast(::rocjitsu::amdgpu::read_vop3_true16_src( - src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0)), + static_cast( + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1)), + inst_.clamp))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -10714,10 +10726,10 @@ void VAddNcI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -10747,10 +10759,10 @@ RJ_NOINLINE void VAddNcI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) + - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -10768,10 +10780,10 @@ void VSubNcI16Vop3::execute_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } @@ -10801,10 +10813,10 @@ RJ_NOINLINE void VSubNcI16Vop3::execute_modifier_impl(amdgpu::Wavefront &wf) { if (!(exec & (1ULL << lane))) continue; { - uint32_t src_half = static_cast( - static_cast(static_cast(static_cast(static_cast( - (::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0) - - ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1))))))); + uint32_t src_half = static_cast(static_cast(static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + ::rocjitsu::amdgpu::read_vop3_true16_src(src0, wf, lane, opsel, 0), + ::rocjitsu::amdgpu::read_vop3_true16_src(src1, wf, lane, opsel, 1), inst_.clamp)))))); ::rocjitsu::amdgpu::write_vop3_true16_dst(vdst, wf, lane, opsel, src_half, true); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3p_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3p_exec.cpp index 8ebb2ccfcd42..66b169592f8e 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3p_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3p_exec.cpp @@ -44,8 +44,12 @@ void VPkMadI16Vop3p::execute_impl(amdgpu::Wavefront &wf) { int16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); int16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); int16_t c_hi = static_cast(sel2_hi ? (raw2 >> 16) : raw2); - uint16_t rlo = static_cast(static_cast(a_lo) * b_lo + c_lo); - uint16_t rhi = static_cast(static_cast(a_hi) * b_hi + c_hi); + uint16_t rlo = amdgpu::vop3_integer_mad(static_cast(a_lo), + static_cast(b_lo), + static_cast(c_lo), inst_.clamp); + uint16_t rhi = amdgpu::vop3_integer_mad(static_cast(a_hi), + static_cast(b_hi), + static_cast(c_hi), inst_.clamp); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -88,8 +92,10 @@ void VPkAddI16Vop3p::execute_impl(amdgpu::Wavefront &wf) { int16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); int16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); int16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo + b_lo); - uint16_t rhi = static_cast(a_hi + b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_add(a_lo, b_lo, inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_add(a_hi, b_hi, inst_.clamp)); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -110,8 +116,10 @@ void VPkSubI16Vop3p::execute_impl(amdgpu::Wavefront &wf) { int16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); int16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); int16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo - b_lo); - uint16_t rhi = static_cast(a_hi - b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_sub(a_lo, b_lo, inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_sub(a_hi, b_hi, inst_.clamp)); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -247,8 +255,8 @@ void VPkMadU16Vop3p::execute_impl(amdgpu::Wavefront &wf) { uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); uint16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); uint16_t c_hi = static_cast(sel2_hi ? (raw2 >> 16) : raw2); - uint16_t rlo = static_cast(static_cast(a_lo) * b_lo + c_lo); - uint16_t rhi = static_cast(static_cast(a_hi) * b_hi + c_hi); + uint16_t rlo = amdgpu::vop3_integer_mad(a_lo, b_lo, c_lo, inst_.clamp); + uint16_t rhi = amdgpu::vop3_integer_mad(a_hi, b_hi, c_hi, inst_.clamp); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -269,8 +277,10 @@ void VPkAddU16Vop3p::execute_impl(amdgpu::Wavefront &wf) { uint16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); uint16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo + b_lo); - uint16_t rhi = static_cast(a_hi + b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_add(a_lo, b_lo, inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_add(a_hi, b_hi, inst_.clamp)); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -291,8 +301,10 @@ void VPkSubU16Vop3p::execute_impl(amdgpu::Wavefront &wf) { uint16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); uint16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo - b_lo); - uint16_t rhi = static_cast(a_hi - b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_sub(a_lo, b_lo, inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_sub(a_hi, b_hi, inst_.clamp)); amdgpu::sdwa::write_lane( *this, wf, vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h index 007db517045e..2561e25067d5 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h @@ -2946,7 +2946,7 @@ inline void execute_v_add_co_ci_u32_vop3([[maybe_unused]] Inst &inst, vcc |= (1ULL << lane); else vcc &= ~(1ULL << lane); - return static_cast(w); + return inst.inst_.clamp && w > 0xFFFFFFFFULL ? UINT32_MAX : static_cast(w); }()); } commit_result(vcc); @@ -2996,7 +2996,7 @@ inline void execute_v_add_co_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse vcc |= (1ULL << lane); else vcc &= ~(1ULL << lane); - return static_cast(w); + return inst.inst_.clamp && w > 0xFFFFFFFFULL ? UINT32_MAX : static_cast(w); }()); } commit_result(vcc); @@ -3217,9 +3217,11 @@ inline void execute_v_add_i32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) + - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_add(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + inst.inst_.clamp)); } } @@ -3246,10 +3248,12 @@ inline void execute_v_add_nc_i16_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - static_cast(static_cast(static_cast( - (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) + - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)))))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + static_cast( + static_cast(static_cast(amdgpu::vop3_integer_add( + amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), inst.inst_.clamp))))); } } @@ -3260,9 +3264,11 @@ inline void execute_v_add_nc_i32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) + - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_add(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + inst.inst_.clamp)); } } @@ -3274,9 +3280,10 @@ inline void execute_v_add_nc_u16_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse continue; sdwa::write_lane( inst, wf, inst.vdst, lane, - static_cast(static_cast( - (static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)) + - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)))))); + static_cast(static_cast(amdgpu::vop3_integer_add( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.clamp)))); } } @@ -3300,9 +3307,11 @@ inline void execute_v_add_nc_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) + - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_add(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + inst.inst_.clamp)); } } @@ -3341,9 +3350,11 @@ inline void execute_v_add_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) + - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_add(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + inst.inst_.clamp)); } } @@ -3400,7 +3411,7 @@ inline void execute_v_addc_co_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unus vcc |= (1ULL << lane); else vcc &= ~(1ULL << lane); - return static_cast(w); + return inst.inst_.clamp && w > 0xFFFFFFFFULL ? UINT32_MAX : static_cast(w); }()); } commit_result(vcc); @@ -13190,11 +13201,16 @@ inline void execute_v_mad_co_i64_i32_vop3([[maybe_unused]] Inst &inst, int64_t s0 = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); int64_t s1 = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(inst.src2, lane); - uint64_t product = static_cast(s0) * static_cast(s1); + uint64_t product = static_cast(s0 * s1); uint64_t result = product + s2; - constexpr uint64_t sign_bit = 1ULL << 63; - if (((~(product ^ s2) & (product ^ result)) & sign_bit) != 0) + bool overflow = amdgpu::signed_add_overflows(product, s2); + if (overflow) carry |= 1ULL << lane; + if (inst.inst_.clamp && overflow) { + result = (product & (1ULL << 63)) + ? std::bit_cast(std::numeric_limits::min()) + : static_cast(std::numeric_limits::max()); + } sdwa::write_lane64(inst, wf, inst.vdst, lane, result); } commit_result(carry); @@ -13214,8 +13230,11 @@ inline void execute_v_mad_co_u64_u32_vop3([[maybe_unused]] Inst &inst, uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(inst.src2, lane); uint64_t product = s0 * s1; uint64_t result = product + s2; - if (result < product) + bool overflow = result < product; + if (overflow) carry |= 1ULL << lane; + if (inst.inst_.clamp && overflow) + result = std::numeric_limits::max(); sdwa::write_lane64(inst, wf, inst.vdst, lane, result); } commit_result(carry); @@ -13285,10 +13304,12 @@ inline void execute_v_mad_i16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] continue; sdwa::write_lane( inst, wf, inst.vdst, lane, - static_cast(static_cast(static_cast( - ((static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)) * - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))) + - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane))))))); + static_cast( + static_cast(static_cast(amdgpu::vop3_integer_mad( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane)), + inst.inst_.clamp))))); } } @@ -13310,9 +13331,10 @@ inline void execute_v_mad_i32_i16_vop3([[maybe_unused]] Inst &inst, int32_t s0 = static_cast(s0_raw); int32_t s1 = static_cast(s1_raw); int32_t s2 = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane)); - sdwa::write_lane(inst, wf, inst.vdst, lane, - static_cast(s0) * static_cast(s1) + - static_cast(s2)); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_mad(static_cast(s0), static_cast(s1), + static_cast(s2), inst.inst_.clamp)); } } @@ -13328,9 +13350,10 @@ inline void execute_v_mad_i32_i24_vop3([[maybe_unused]] Inst &inst, continue; sdwa::write_lane( inst, wf, inst.vdst, lane, - ::rocjitsu::amdgpu::mad_i24_u32(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), - amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane))); + amdgpu::vop3_integer_mad(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane), + inst.inst_.clamp)); } } @@ -13356,11 +13379,16 @@ inline void execute_v_mad_i64_i32_vop3([[maybe_unused]] Inst &inst, [[maybe_unus int64_t s0 = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); int64_t s1 = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(inst.src2, lane); - uint64_t product = static_cast(s0) * static_cast(s1); + uint64_t product = static_cast(s0 * s1); uint64_t result = product + s2; - constexpr uint64_t sign_bit = 1ULL << 63; - if (((~(product ^ s2) & (product ^ result)) & sign_bit) != 0) + bool overflow = amdgpu::signed_add_overflows(product, s2); + if (overflow) carry |= 1ULL << lane; + if (inst.inst_.clamp && overflow) { + result = (product & (1ULL << 63)) + ? std::bit_cast(std::numeric_limits::min()) + : static_cast(std::numeric_limits::max()); + } sdwa::write_lane64(inst, wf, inst.vdst, lane, result); } commit_result(carry); @@ -13559,10 +13587,10 @@ inline void execute_v_mad_u16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] continue; sdwa::write_lane( inst, wf, inst.vdst, lane, - static_cast(static_cast(::rocjitsu::amdgpu::mad_lo_u16( + static_cast(static_cast(amdgpu::vop3_integer_mad( amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), - amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane))))); + amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane), inst.inst_.clamp)))); } } @@ -13579,7 +13607,8 @@ inline void execute_v_mad_u32_u16_vop3([[maybe_unused]] Inst &inst, uint32_t s0 = ::rocjitsu::amdgpu::read_vop3_true16_src(inst.src0, wf, lane, opsel, 0); uint32_t s1 = ::rocjitsu::amdgpu::read_vop3_true16_src(inst.src1, wf, lane, opsel, 1); uint32_t s2 = amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane); - sdwa::write_lane(inst, wf, inst.vdst, lane, s0 * s1 + s2); + sdwa::write_lane(inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_mad(s0, s1, s2, inst.inst_.clamp)); } } @@ -13592,11 +13621,12 @@ inline void execute_v_mad_u32_u24_vop3([[maybe_unused]] Inst &inst, for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane( - inst, wf, inst.vdst, lane, - ((amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) & 0x00FFFFFFu) * - (amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane) & 0x00FFFFFFu) + - amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane))); + sdwa::write_lane(inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_mad( + amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane), + inst.inst_.clamp)); } } @@ -13619,8 +13649,11 @@ inline void execute_v_mad_u64_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unus uint64_t s2 = amdgpu::RegisterAccess(wf).read_lane64(inst.src2, lane); uint64_t product = s0 * s1; uint64_t result = product + s2; - if (result < product) + bool overflow = result < product; + if (overflow) carry |= 1ULL << lane; + if (inst.inst_.clamp && overflow) + result = std::numeric_limits::max(); sdwa::write_lane64(inst, wf, inst.vdst, lane, result); } commit_result(carry); @@ -16266,20 +16299,12 @@ inline void execute_v_msad_u8_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { - auto a = amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane); - auto b = amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane); - auto c = amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane); - uint32_t r = c; - for (int i = 0; i < 4; ++i) { - uint8_t value = (a >> (i * 8)) & 0xFF; - uint8_t reference = (b >> (i * 8)) & 0xFF; - if (reference != 0) { - r += (value > reference) ? (value - reference) : (reference - value); - } - } - return r; - }()); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_msad_u8(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane), + inst.inst_.clamp)); } } @@ -16721,8 +16746,9 @@ inline void execute_v_mul_i32_i24_vop3([[maybe_unused]] Inst &inst, continue; sdwa::write_lane( inst, wf, inst.vdst, lane, - ::rocjitsu::amdgpu::mul_i24_u32(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); + amdgpu::vop3_integer_mul(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + inst.inst_.clamp)); } } @@ -16873,10 +16899,11 @@ inline void execute_v_mul_u32_u24_vop3([[maybe_unused]] Inst &inst, for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane( - inst, wf, inst.vdst, lane, - ((amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) & 0x00FFFFFFu) * - (amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane) & 0x00FFFFFFu))); + sdwa::write_lane(inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_mul( + amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + inst.inst_.clamp)); } } @@ -17164,8 +17191,10 @@ inline void execute_v_pk_add_i16_vop3p([[maybe_unused]] Inst &inst, int16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); int16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); int16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo + b_lo); - uint16_t rhi = static_cast(a_hi + b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_add(a_lo, b_lo, inst.inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_add(a_hi, b_hi, inst.inst_.clamp)); sdwa::write_lane(inst, wf, inst.vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -17193,8 +17222,10 @@ inline void execute_v_pk_add_u16_vop3p([[maybe_unused]] Inst &inst, uint16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); uint16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo + b_lo); - uint16_t rhi = static_cast(a_hi + b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_add(a_lo, b_lo, inst.inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_add(a_hi, b_hi, inst.inst_.clamp)); sdwa::write_lane(inst, wf, inst.vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -17467,8 +17498,12 @@ inline void execute_v_pk_mad_i16_vop3p([[maybe_unused]] Inst &inst, int16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); int16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); int16_t c_hi = static_cast(sel2_hi ? (raw2 >> 16) : raw2); - uint16_t rlo = static_cast(static_cast(a_lo) * b_lo + c_lo); - uint16_t rhi = static_cast(static_cast(a_hi) * b_hi + c_hi); + uint16_t rlo = amdgpu::vop3_integer_mad( + static_cast(a_lo), static_cast(b_lo), static_cast(c_lo), + inst.inst_.clamp); + uint16_t rhi = amdgpu::vop3_integer_mad( + static_cast(a_hi), static_cast(b_hi), static_cast(c_hi), + inst.inst_.clamp); sdwa::write_lane(inst, wf, inst.vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -17507,8 +17542,8 @@ inline void execute_v_pk_mad_u16_vop3p([[maybe_unused]] Inst &inst, uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); uint16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); uint16_t c_hi = static_cast(sel2_hi ? (raw2 >> 16) : raw2); - uint16_t rlo = static_cast(static_cast(a_lo) * b_lo + c_lo); - uint16_t rhi = static_cast(static_cast(a_hi) * b_hi + c_hi); + uint16_t rlo = amdgpu::vop3_integer_mad(a_lo, b_lo, c_lo, inst.inst_.clamp); + uint16_t rhi = amdgpu::vop3_integer_mad(a_hi, b_hi, c_hi, inst.inst_.clamp); sdwa::write_lane(inst, wf, inst.vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -17882,8 +17917,10 @@ inline void execute_v_pk_sub_i16_vop3p([[maybe_unused]] Inst &inst, int16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); int16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); int16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo - b_lo); - uint16_t rhi = static_cast(a_hi - b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_sub(a_lo, b_lo, inst.inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_sub(a_hi, b_hi, inst.inst_.clamp)); sdwa::write_lane(inst, wf, inst.vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -17911,8 +17948,10 @@ inline void execute_v_pk_sub_u16_vop3p([[maybe_unused]] Inst &inst, uint16_t b_lo = static_cast(sel1_lo ? (raw1 >> 16) : raw1); uint16_t a_hi = static_cast(sel0_hi ? (raw0 >> 16) : raw0); uint16_t b_hi = static_cast(sel1_hi ? (raw1 >> 16) : raw1); - uint16_t rlo = static_cast(a_lo - b_lo); - uint16_t rhi = static_cast(a_hi - b_hi); + uint16_t rlo = + static_cast(amdgpu::vop3_integer_sub(a_lo, b_lo, inst.inst_.clamp)); + uint16_t rhi = + static_cast(amdgpu::vop3_integer_sub(a_hi, b_hi, inst.inst_.clamp)); sdwa::write_lane(inst, wf, inst.vdst, lane, static_cast(rlo) | (static_cast(rhi) << 16)); } @@ -18550,18 +18589,12 @@ inline void execute_v_sad_hi_u8_vop3([[maybe_unused]] Inst &inst, [[maybe_unused for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { - auto a = amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane); - auto b = amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane); - auto c = amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane); - uint32_t r = 0; - for (int i = 0; i < 4; ++i) { - uint8_t ab = (a >> (i * 8)) & 0xFF; - uint8_t bb = (b >> (i * 8)) & 0xFF; - r += (ab > bb) ? (ab - bb) : (bb - ab); - } - return (r << 16) + c; - }()); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_sad_hi_u8(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane), + inst.inst_.clamp)); } } @@ -18581,16 +18614,12 @@ inline void execute_v_sad_u16_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { - auto a = amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane); - auto b = amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane); - auto c = amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane); - uint16_t al = a & 0xFFFF; - uint16_t ah = (a >> 16) & 0xFFFF; - uint16_t bl = b & 0xFFFF; - uint16_t bh = (b >> 16) & 0xFFFF; - return c + ((al > bl) ? (al - bl) : (bl - al)) + ((ah > bh) ? (ah - bh) : (bh - ah)); - }()); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_sad_u16(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane), + inst.inst_.clamp)); } } @@ -18605,12 +18634,12 @@ inline void execute_v_sad_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { - auto a = amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane); - auto b = amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane); - auto c = amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane); - return c + ((a > b) ? (a - b) : (b - a)); - }()); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_sad_u32(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane), + inst.inst_.clamp)); } } @@ -18622,18 +18651,12 @@ inline void execute_v_sad_u8_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { - auto a = amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane); - auto b = amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane); - auto c = amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane); - uint32_t r = c; - for (int i = 0; i < 4; ++i) { - uint8_t ab = (a >> (i * 8)) & 0xFF; - uint8_t bb = (b >> (i * 8)) & 0xFF; - r += (ab > bb) ? (ab - bb) : (bb - ab); - } - return r; - }()); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_sad_u8(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src2, lane), + inst.inst_.clamp)); } } @@ -19009,14 +19032,14 @@ inline void execute_v_sub_co_ci_u32_vop3([[maybe_unused]] Inst &inst, continue; sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { uint64_t a = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), - b = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), - c = static_cast( - ((amdgpu::read_wave_mask_scalar(inst.src2, wf) >> lane) & 1)); - if (a < b + c) + b = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)) + + static_cast( + ((amdgpu::read_wave_mask_scalar(inst.src2, wf) >> lane) & 1)); + if (a < b) vcc |= (1ULL << lane); else vcc &= ~(1ULL << lane); - return static_cast(a - b - c); + return inst.inst_.clamp && a < b ? 0u : static_cast(a - b); }()); } commit_result(vcc); @@ -19056,13 +19079,13 @@ inline void execute_v_sub_co_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse if (!(exec & (1ULL << lane))) continue; sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { - uint32_t a = amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), - b = amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane); + uint64_t a = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + b = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)); if (a < b) vcc |= (1ULL << lane); else vcc &= ~(1ULL << lane); - return a - b; + return inst.inst_.clamp && a < b ? 0u : static_cast(a - b); }()); } commit_result(vcc); @@ -19222,9 +19245,11 @@ inline void execute_v_sub_i32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) - - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_sub(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + inst.inst_.clamp)); } } @@ -19234,10 +19259,12 @@ inline void execute_v_sub_nc_i16_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - static_cast(static_cast(static_cast( - (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) - - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)))))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + static_cast( + static_cast(static_cast(amdgpu::vop3_integer_sub( + amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), inst.inst_.clamp))))); } } @@ -19248,9 +19275,11 @@ inline void execute_v_sub_nc_i32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) - - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_sub(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + inst.inst_.clamp)); } } @@ -19262,9 +19291,10 @@ inline void execute_v_sub_nc_u16_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse continue; sdwa::write_lane( inst, wf, inst.vdst, lane, - static_cast(static_cast( - (static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)) - - static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)))))); + static_cast(static_cast(amdgpu::vop3_integer_sub( + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), + static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + inst.inst_.clamp)))); } } @@ -19288,9 +19318,11 @@ inline void execute_v_sub_nc_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) - - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_sub(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + inst.inst_.clamp)); } } @@ -19329,9 +19361,11 @@ inline void execute_v_sub_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unused]] for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - (amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane) - - amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_sub(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + inst.inst_.clamp)); } } @@ -19381,14 +19415,14 @@ inline void execute_v_subb_co_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unus continue; sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { uint64_t a = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), - b = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), - c = static_cast( - ((amdgpu::read_wave_mask_scalar(inst.src2, wf) >> lane) & 1)); - if (a < b + c) + b = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)) + + static_cast( + ((amdgpu::read_wave_mask_scalar(inst.src2, wf) >> lane) & 1)); + if (a < b) vcc |= (1ULL << lane); else vcc &= ~(1ULL << lane); - return static_cast(a - b - c); + return inst.inst_.clamp && a < b ? 0u : static_cast(a - b); }()); } commit_result(vcc); @@ -19442,14 +19476,14 @@ inline void execute_v_subbrev_co_u32_vop3([[maybe_unused]] Inst &inst, continue; sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { uint64_t a = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), - b = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), - c = static_cast( - ((amdgpu::read_wave_mask_scalar(inst.src2, wf) >> lane) & 1)); - if (a < b + c) + b = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)) + + static_cast( + ((amdgpu::read_wave_mask_scalar(inst.src2, wf) >> lane) & 1)); + if (a < b) vcc |= (1ULL << lane); else vcc &= ~(1ULL << lane); - return static_cast(a - b - c); + return inst.inst_.clamp && a < b ? 0u : static_cast(a - b); }()); } commit_result(vcc); @@ -19503,14 +19537,14 @@ inline void execute_v_subrev_co_ci_u32_vop3([[maybe_unused]] Inst &inst, continue; sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { uint64_t a = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), - b = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)), - c = static_cast( - ((amdgpu::read_wave_mask_scalar(inst.src2, wf) >> lane) & 1)); - if (a < b + c) + b = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)) + + static_cast( + ((amdgpu::read_wave_mask_scalar(inst.src2, wf) >> lane) & 1)); + if (a < b) vcc |= (1ULL << lane); else vcc &= ~(1ULL << lane); - return static_cast(a - b - c); + return inst.inst_.clamp && a < b ? 0u : static_cast(a - b); }()); } commit_result(vcc); @@ -19552,13 +19586,13 @@ inline void execute_v_subrev_co_u32_vop3([[maybe_unused]] Inst &inst, if (!(exec & (1ULL << lane))) continue; sdwa::write_lane(inst, wf, inst.vdst, lane, [&]() { - uint32_t a = amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), - b = amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane); + uint64_t a = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane)), + b = static_cast(amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane)); if (a < b) vcc |= (1ULL << lane); else vcc &= ~(1ULL << lane); - return a - b; + return inst.inst_.clamp && a < b ? 0u : static_cast(a - b); }()); } commit_result(vcc); @@ -19733,9 +19767,11 @@ inline void execute_v_subrev_nc_u32_vop3([[maybe_unused]] Inst &inst, for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - (amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane) - - amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_sub(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + inst.inst_.clamp)); } } @@ -19774,9 +19810,11 @@ inline void execute_v_subrev_u32_vop3([[maybe_unused]] Inst &inst, [[maybe_unuse for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; - sdwa::write_lane(inst, wf, inst.vdst, lane, - (amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane) - - amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane))); + sdwa::write_lane( + inst, wf, inst.vdst, lane, + amdgpu::vop3_integer_sub(amdgpu::RegisterAccess(wf).read_lane(inst.src1, lane), + amdgpu::RegisterAccess(wf).read_lane(inst.src0, lane), + inst.inst_.clamp)); } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h index 7180bb28d1ef..1619bec00db1 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_glue.h @@ -24,6 +24,7 @@ #include "simdojo/components/vector_reg.h" #include "util/simd.h" +#include #include #include #include @@ -94,6 +95,151 @@ template inline T clamp_floating_result(T value, const Wavefront &w return clamp_floating_result(value, floating_clamp_nan_to_zero(wf)); } +/// @brief Execute VOP3 integer addition, saturating when CLAMP is set. +template +inline std::make_unsigned_t vop3_integer_add(std::make_unsigned_t lhs, + std::make_unsigned_t rhs, bool clamp) { + static_assert(std::is_integral_v && (sizeof(T) == 2 || sizeof(T) == 4 || sizeof(T) == 8)); + using U = std::make_unsigned_t; + const U lhs_bits = lhs; + const U rhs_bits = rhs; + if (!clamp) + return static_cast(lhs_bits + rhs_bits); + if constexpr (std::is_signed_v) { + static_assert(sizeof(T) <= 4, "signed 64-bit VOP3 saturation is not implemented"); + using Wide = std::conditional_t; + const Wide wide = static_cast(std::bit_cast(lhs_bits)) + + static_cast(std::bit_cast(rhs_bits)); + const Wide saturated = std::clamp(wide, static_cast(std::numeric_limits::min()), + static_cast(std::numeric_limits::max())); + return static_cast(saturated); + } else { + const U max = std::numeric_limits::max(); + return rhs_bits > max - lhs_bits ? max : static_cast(lhs_bits + rhs_bits); + } +} + +/// @brief Add with intrinsic saturation, then select the maximum or minimum operand. +template +inline std::make_unsigned_t vop3_integer_add_minmax(std::make_unsigned_t lhs, + std::make_unsigned_t rhs, + std::make_unsigned_t bound) { + static_assert(std::is_integral_v && sizeof(T) == 4); + using U = std::make_unsigned_t; + const U sum_bits = vop3_integer_add(lhs, rhs, true); + if constexpr (std::is_signed_v) { + const T sum = std::bit_cast(sum_bits); + const T typed_bound = std::bit_cast(bound); + return static_cast(SelectMax ? std::max(sum, typed_bound) : std::min(sum, typed_bound)); + } else { + return SelectMax ? std::max(sum_bits, bound) : std::min(sum_bits, bound); + } +} + +/// @brief Execute an integer multiply-add with an exact intermediate. +template +inline std::make_unsigned_t vop3_integer_mad(uint32_t lhs, uint32_t rhs, + std::make_unsigned_t addend, bool clamp) { + static_assert(std::is_integral_v && sizeof(T) <= 4); + static_assert(SourceBits == 16 || SourceBits == 24); + using U = std::make_unsigned_t; + if constexpr (std::is_signed_v) { + const auto extend = [](uint32_t value) -> int64_t { + constexpr uint32_t shift = 32 - SourceBits; + return static_cast(static_cast(value << shift) >> shift); + }; + const int64_t wide = extend(lhs) * extend(rhs) + static_cast(std::bit_cast(addend)); + if (!clamp) + return static_cast(wide); + return static_cast(std::clamp(wide, static_cast(std::numeric_limits::min()), + static_cast(std::numeric_limits::max()))); + } else { + constexpr uint32_t source_mask = (uint32_t{1} << SourceBits) - 1u; + const uint64_t wide = static_cast(lhs & source_mask) * (rhs & source_mask) + addend; + if (clamp && wide > std::numeric_limits::max()) + return std::numeric_limits::max(); + return static_cast(wide); + } +} + +template +inline std::make_unsigned_t vop3_integer_mul(uint32_t lhs, uint32_t rhs, bool clamp) { + return vop3_integer_mad(lhs, rhs, std::make_unsigned_t{0}, clamp); +} + +inline uint32_t vop3_integer_sad_u8(uint32_t lhs, uint32_t rhs, uint32_t addend, bool clamp) { + uint32_t difference = 0; + for (unsigned i = 0; i < 4; ++i) { + const uint32_t a = (lhs >> (i * 8)) & 0xffu; + const uint32_t b = (rhs >> (i * 8)) & 0xffu; + difference += a > b ? a - b : b - a; + } + return vop3_integer_add(difference, addend, clamp); +} + +/// @brief Execute shifted byte SAD plus accumulation, saturating when CLAMP is set. +inline uint32_t vop3_integer_sad_hi_u8(uint32_t lhs, uint32_t rhs, uint32_t addend, bool clamp) { + uint32_t difference = 0; + for (unsigned i = 0; i < 4; ++i) { + const uint32_t a = (lhs >> (i * 8)) & 0xffu; + const uint32_t b = (rhs >> (i * 8)) & 0xffu; + difference += a > b ? a - b : b - a; + } + const uint64_t wide = (static_cast(difference) << 16) + addend; + if (clamp && wide > UINT32_MAX) + return UINT32_MAX; + return static_cast(wide); +} + +inline uint32_t vop3_integer_sad_u16(uint32_t lhs, uint32_t rhs, uint32_t addend, bool clamp) { + uint32_t difference = 0; + for (unsigned i = 0; i < 2; ++i) { + const uint32_t a = (lhs >> (i * 16)) & 0xffffu; + const uint32_t b = (rhs >> (i * 16)) & 0xffffu; + difference += a > b ? a - b : b - a; + } + return vop3_integer_add(difference, addend, clamp); +} + +inline uint32_t vop3_integer_sad_u32(uint32_t lhs, uint32_t rhs, uint32_t addend, bool clamp) { + const uint32_t difference = lhs > rhs ? lhs - rhs : rhs - lhs; + return vop3_integer_add(difference, addend, clamp); +} + +inline uint32_t vop3_integer_msad_u8(uint32_t lhs, uint32_t rhs, uint32_t addend, bool clamp) { + uint32_t difference = 0; + for (unsigned i = 0; i < 4; ++i) { + const uint32_t a = (lhs >> (i * 8)) & 0xffu; + const uint32_t b = (rhs >> (i * 8)) & 0xffu; + if (b != 0) + difference += a > b ? a - b : b - a; + } + return vop3_integer_add(difference, addend, clamp); +} + +/// @brief Execute VOP3 integer subtraction, saturating when CLAMP is set. +template +inline std::make_unsigned_t vop3_integer_sub(std::make_unsigned_t lhs, + std::make_unsigned_t rhs, bool clamp) { + static_assert(std::is_integral_v && (sizeof(T) == 2 || sizeof(T) == 4 || sizeof(T) == 8)); + using U = std::make_unsigned_t; + const U lhs_bits = lhs; + const U rhs_bits = rhs; + if (!clamp) + return static_cast(lhs_bits - rhs_bits); + if constexpr (std::is_signed_v) { + static_assert(sizeof(T) <= 4, "signed 64-bit VOP3 saturation is not implemented"); + using Wide = std::conditional_t; + const Wide wide = static_cast(std::bit_cast(lhs_bits)) - + static_cast(std::bit_cast(rhs_bits)); + const Wide saturated = std::clamp(wide, static_cast(std::numeric_limits::min()), + static_cast(std::numeric_limits::max())); + return static_cast(saturated); + } else { + return lhs_bits < rhs_bits ? U{0} : static_cast(lhs_bits - rhs_bits); + } +} + /// @brief Return whether V_DOT4 integer instructions honor the encoded CLAMP bit. /// @details GFX12 (RDNA4) and the CDNA5-backed gfx1250 profile explicitly ignore CLAMP /// for the integer DOT4 forms; earlier CDNA/RDNA profiles retain the ordinary @@ -1675,15 +1821,14 @@ template /// VOP3 integer/bitwise binary SIMD fast path. Same shape as /// try_execute_binary_vop2_simd but reads the VOP3 operands `src0`/`src1` /// (instead of `src0`/`vsrc1`). The generated integer/bitwise VOP3 bodies apply -/// no source/result modifiers (abs/neg/omod are float-only; clamp on an integer -/// op means saturate, which these wrap-around/bitwise twins do not request), so -/// the plain op is bit-identical to the scalar body on every input. T is a -/// 32-bit integer lane type. +/// no source/result modifiers other than integer CLAMP. Saturating integer +/// arithmetic falls back to the scalar body; the plain op is bit-identical to +/// that body when CLAMP is clear. T is a 32-bit integer lane type. template requires(util::has_stdx_simd) [[nodiscard]] inline bool try_execute_binary_vop3_simd(Inst &inst, Wavefront &wf, BinOp bin_op) { - if (simd_force_scalar() || !sdwa::supports_direct_simd_store(inst) || !inst.src0.simd_capable() || - !inst.src1.simd_capable() || !inst.vdst.simd_capable()) + if (inst.inst_.clamp != 0u || simd_force_scalar() || !sdwa::supports_direct_simd_store(inst) || + !inst.src0.simd_capable() || !inst.src1.simd_capable() || !inst.vdst.simd_capable()) return false; constexpr std::size_t W = util::native_width_v; const uint64_t chunk_full = util::mask(static_cast(W)); @@ -2232,16 +2377,17 @@ template /// VOP3 integer/bitwise ternary SIMD fast path. Reads `src0`/`src1`/`src2`, /// runs `tern_op(a, b, c)`, and masked-stores the result. The generated scalar -/// bodies for these ternary integer ops apply no source/result modifiers (abs/ -/// neg/omod are float-only; clamp is unused on integer 3-source ops), so the -/// plain functor is bit-identical to the scalar body. T is a 32-bit integer -/// lane type (typically uint32_t). Same SIMD-capable / EXEC-chunk loop as the -/// binary VOP3 path. +/// bodies for these ternary integer ops apply no source/result modifiers except +/// integer CLAMP. Saturating arithmetic falls back to the scalar body; the plain +/// functor is bit-identical to that body when CLAMP is clear. T is a 32-bit +/// integer lane type (typically uint32_t). Same SIMD-capable / EXEC-chunk loop +/// as the binary VOP3 path. template requires(util::has_stdx_simd) [[nodiscard]] inline bool try_execute_ternary_vop3_simd(Inst &inst, Wavefront &wf, TernOp tern_op) { - if (simd_force_scalar() || !sdwa::supports_direct_simd_store(inst) || !inst.src0.simd_capable() || - !inst.src1.simd_capable() || !inst.src2.simd_capable() || !inst.vdst.simd_capable()) + if (inst.inst_.clamp != 0u || simd_force_scalar() || !sdwa::supports_direct_simd_store(inst) || + !inst.src0.simd_capable() || !inst.src1.simd_capable() || !inst.src2.simd_capable() || + !inst.vdst.simd_capable()) return false; constexpr std::size_t W = util::native_width_v; const uint64_t chunk_full = util::mask(static_cast(W)); @@ -2277,8 +2423,9 @@ template [[nodiscard]] inline bool try_execute_ternary_vop3_true16_src01_simd(Inst &inst, Wavefront &wf, TernOp tern_op) { static_assert(std::is_same_v); - if (simd_force_scalar() || !sdwa::supports_direct_simd_store(inst) || !inst.src0.simd_capable() || - !inst.src1.simd_capable() || !inst.src2.simd_capable() || !inst.vdst.simd_capable()) + if (inst.inst_.clamp != 0u || simd_force_scalar() || !sdwa::supports_direct_simd_store(inst) || + !inst.src0.simd_capable() || !inst.src1.simd_capable() || !inst.src2.simd_capable() || + !inst.vdst.simd_capable()) return false; constexpr std::size_t W = util::native_width_v; const uint64_t chunk_full = util::mask(static_cast(W)); @@ -3281,6 +3428,10 @@ template if (simd_force_scalar() || !inst.src0.simd_capable() || !inst.src1.simd_capable() || !inst.src2.simd_capable() || !inst.vdst.simd_capable()) return false; + if constexpr (requires { inst.inst_.clamp; }) { + if (inst.inst_.clamp) + return false; + } constexpr std::size_t W = util::native_width64; const uint64_t chunk_full = util::mask(static_cast(W)); const uint64_t exec = dpp::execution_lane_mask(inst, wf); @@ -3334,8 +3485,8 @@ template [[nodiscard]] inline bool try_execute_binary_vop3_co_result_simd(Inst &inst, Wavefront &wf, CarryOp carry_op, WriteResult write_result) { - if (simd_force_scalar() || !inst.src0.simd_capable() || !inst.src1.simd_capable() || - !inst.vdst.simd_capable()) + if (simd_force_scalar() || inst.inst_.clamp || !inst.src0.simd_capable() || + !inst.src1.simd_capable() || !inst.vdst.simd_capable()) return false; using T = uint32_t; constexpr std::size_t W = util::native_width_v; @@ -3387,8 +3538,8 @@ template [[nodiscard]] inline bool try_execute_binary_vop3_cin_result_simd(Inst &inst, Wavefront &wf, CarryOp carry_op, WriteResult write_result) { - if (simd_force_scalar() || !inst.src0.simd_capable() || !inst.src1.simd_capable() || - !inst.vdst.simd_capable()) + if (simd_force_scalar() || inst.inst_.clamp || !inst.src0.simd_capable() || + !inst.src1.simd_capable() || !inst.vdst.simd_capable()) return false; using T = uint32_t; constexpr std::size_t W = util::native_width_v; @@ -3571,8 +3722,8 @@ template /// output) and op_sel_hi = 3 (both srcs feed their high half into the /// high output) — the LLVM-AS encoder emits this for the default-mode /// pk mnemonics, and the SIMD fast path bails when any other combination -/// is requested. The scalar bodies do NOT apply neg / neg_hi / clamp on -/// these integer pk ops, so the SIMD path also passes through. Functor +/// or integer saturation is requested. The scalar bodies apply CLAMP to +/// packed integer add/sub independently for each selected half. Functor /// receives the two source u32 lane vectors (each holding {low16, high16} /// packed) and returns the same shape; the per-half decompose / recompose /// lives inside the functor for op-specific flexibility (e.g. mul_lo @@ -3583,7 +3734,7 @@ template if (simd_force_scalar() || !sdwa::supports_direct_simd_store(inst) || !inst.src0.simd_capable() || !inst.src1.simd_capable() || !inst.vdst.simd_capable()) return false; - if (inst.inst_.op_sel != 0u || inst.inst_.op_sel_hi != 3u) + if (inst.inst_.clamp || inst.inst_.op_sel != 0u || inst.inst_.op_sel_hi != 3u) return false; using T = uint32_t; constexpr std::size_t W = util::native_width_v; @@ -3615,16 +3766,17 @@ template /// third source's high-half selector op_sel_hi_2 == 1 (per the scalar body /// `sel2_hi = inst.inst_.op_sel_hi_2`, which is a single bit — value 1 picks /// the high half for the high-output computation). For pk_mad_i16/u16 the -/// scalar bodies also do not apply neg/neg_hi/clamp; the SIMD path passes -/// through. Functor receives three u32 packed lane vectors and returns the -/// per-half-masked packed result. +/// scalar bodies do not apply neg/neg_hi. Clamped integer MAD falls back to +/// scalar execution for exact per-half saturation. Functor receives three u32 +/// packed lane vectors and returns the per-half-masked packed result. template requires(util::has_stdx_simd) [[nodiscard]] inline bool try_execute_vop3p_pk_ternary_int_simd(Inst &inst, Wavefront &wf, Op op) { if (simd_force_scalar() || !sdwa::supports_direct_simd_store(inst) || !inst.src0.simd_capable() || !inst.src1.simd_capable() || !inst.src2.simd_capable() || !inst.vdst.simd_capable()) return false; - if (inst.inst_.op_sel != 0u || inst.inst_.op_sel_hi != 3u || inst.inst_.op_sel_hi_2 != 1u) + if (inst.inst_.clamp || inst.inst_.op_sel != 0u || inst.inst_.op_sel_hi != 3u || + inst.inst_.op_sel_hi_2 != 1u) return false; using T = uint32_t; constexpr std::size_t W = util::native_width_v; diff --git a/emulation/rocjitsu/tests/cdna5_execution_test.cpp b/emulation/rocjitsu/tests/cdna5_execution_test.cpp index d409f6bd8b1e..14fbba7473c4 100644 --- a/emulation/rocjitsu/tests/cdna5_execution_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_execution_test.cpp @@ -156,6 +156,504 @@ TEST(Gfx1250ExecutionTest, GenericF16OmodStaysActiveAndFinalizesWithIeeeDenormMo } } +TEST(Gfx1250ExecutionTest, Vop3IntegerClampSaturatesBeforeNarrowing) { + ForceScalarGuard guard; + for (uint32_t scalar = 0; scalar < 2; ++scalar) { + util::set_force_scalar_for_testing(scalar != 0); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + const uint32_t base = wf->vgpr_alloc().base; + + // Exact instruction from ROCm/rocm-systems#10760: + // v_sub_nc_u32 v4, v4, 4 clamp + cu->write_vgpr(base + 4, 0, 1u); + constexpr std::array kIssueInstruction = {0xd5268004u, 0x02010904u}; + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA5); + ASSERT_NE(decoder, nullptr); + std::unique_ptr issue_inst(decode_valid(*decoder, kIssueInstruction.data())); + ASSERT_NE(issue_inst, nullptr); + EXPECT_EQ(issue_inst->mnemonic(), "v_sub_nc_u32"); + issue_inst->execute(*issue_inst, wf); + EXPECT_EQ(cu->read_vgpr(base + 4, 0), 0u) << "scalar " << scalar; + + struct TestCase { + uint16_t opcode; + uint32_t lhs; + uint32_t rhs; + uint32_t expected; + }; + constexpr std::array kBoundaryCases = {{ + {cdna5::kVAddNcU32Vop3, UINT32_MAX, 1u, UINT32_MAX}, + {cdna5::kVSubNcU32Vop3, 7u, 3u, 4u}, + {cdna5::kVSubrevNcU32Vop3, 3u, 7u, 4u}, + {cdna5::kVSubrevNcU32Vop3, 7u, 3u, 0u}, + {cdna5::kVAddNcI32Vop3, static_cast(INT32_MAX), 1u, + static_cast(INT32_MAX)}, + {cdna5::kVAddNcI32Vop3, static_cast(INT32_MIN), static_cast(-1), + static_cast(INT32_MIN)}, + {cdna5::kVSubNcI32Vop3, static_cast(INT32_MIN), 1u, + static_cast(INT32_MIN)}, + {cdna5::kVSubNcI32Vop3, static_cast(INT32_MAX), static_cast(-1), + static_cast(INT32_MAX)}, + {cdna5::kVAddNcU16Vop3, UINT16_MAX, 1u, UINT16_MAX}, + {cdna5::kVSubNcU16Vop3, 0u, 1u, 0u}, + {cdna5::kVAddNcI16Vop3, static_cast(INT16_MAX), 1u, + static_cast(INT16_MAX)}, + {cdna5::kVSubNcI16Vop3, static_cast(INT16_MIN), 1u, + static_cast(INT16_MIN)}, + }}; + for (const auto &test : kBoundaryCases) { + cu->write_vgpr(base + 0, 0, test.lhs); + cu->write_vgpr(base + 1, 0, test.rhs); + cu->write_vgpr(base + 2, 0, 0u); + const auto words = + cdna5::build_vop3(test.opcode, {.vdst = 2, .clamp = 1, .src0 = 256, .src1 = 257}); + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + inst->execute(*inst, wf); + EXPECT_EQ(cu->read_vgpr(base + 2, 0), test.expected) << "scalar " << scalar; + } + + // CLAMP=0 retains the architectural modulo result. + cu->write_vgpr(base + 0, 0, UINT32_MAX); + cu->write_vgpr(base + 1, 0, 1u); + const auto wrap_words = + cdna5::build_vop3(cdna5::kVAddNcU32Vop3, {.vdst = 2, .src0 = 256, .src1 = 257}); + cdna5::VAddNcU32Vop3 wrap_inst(wrap_words.data()); + wrap_inst.execute_impl(*wf); + EXPECT_EQ(cu->read_vgpr(base + 2, 0), 0u) << "scalar " << scalar; + } +} + +TEST(Gfx1250ExecutionTest, Vop3IntegerMultiplyAccumulateClampSaturatesExactResult) { + ForceScalarGuard guard; + struct TestCase { + uint16_t opcode; + uint32_t src0; + uint32_t src1; + uint32_t src2; + uint32_t expected; + bool clamp = true; + }; + constexpr std::array kCases{ + TestCase{cdna5::kVMulU32U24Vop3, 0x00ffffffu, 0x00ffffffu, 0u, UINT32_MAX}, + TestCase{cdna5::kVMulI32I24Vop3, 0x007fffffu, 0x007fffffu, 0u, + static_cast(INT32_MAX)}, + TestCase{cdna5::kVMulI32I24Vop3, 0x00800000u, 257u, 0u, static_cast(INT32_MIN)}, + TestCase{cdna5::kVMadU32U24Vop3, 0x00ffffffu, 0x00ffffffu, UINT32_MAX, UINT32_MAX}, + TestCase{cdna5::kVMadI32I24Vop3, 0x007fffffu, 0x007fffffu, static_cast(INT32_MAX), + static_cast(INT32_MAX)}, + TestCase{cdna5::kVMadI32I24Vop3, 0x00800000u, 256u, UINT32_MAX, + static_cast(INT32_MIN)}, + TestCase{cdna5::kVMadI32I24Vop3, 0x00800000u, 256u, UINT32_MAX, + static_cast(INT32_MAX), false}, + TestCase{cdna5::kVMadU16Vop3, UINT16_MAX, UINT16_MAX, UINT16_MAX, UINT16_MAX}, + TestCase{cdna5::kVMadI16Vop3, static_cast(INT16_MAX), + static_cast(INT16_MAX), static_cast(INT16_MAX), + static_cast(INT16_MAX)}, + TestCase{cdna5::kVMadI16Vop3, static_cast(INT16_MIN), 2u, static_cast(-1), + static_cast(INT16_MIN)}, + TestCase{cdna5::kVMadU32U16Vop3, UINT16_MAX, UINT16_MAX, UINT32_MAX, UINT32_MAX}, + TestCase{cdna5::kVMadI32I16Vop3, static_cast(INT16_MAX), + static_cast(INT16_MAX), static_cast(INT32_MAX), + static_cast(INT32_MAX)}, + TestCase{cdna5::kVMadI32I16Vop3, static_cast(INT16_MIN), + static_cast(INT16_MAX), static_cast(INT32_MIN), + static_cast(INT32_MIN)}, + TestCase{cdna5::kVAddMaxI32Vop3, static_cast(INT32_MAX), 1u, + static_cast(INT32_MIN), static_cast(INT32_MAX)}, + TestCase{cdna5::kVAddMaxI32Vop3, static_cast(INT32_MAX), 1u, + static_cast(INT32_MIN), static_cast(INT32_MAX), false}, + TestCase{cdna5::kVAddMaxU32Vop3, UINT32_MAX, 1u, 0u, UINT32_MAX}, + TestCase{cdna5::kVAddMaxU32Vop3, UINT32_MAX, 1u, 0u, UINT32_MAX, false}, + TestCase{cdna5::kVAddMinI32Vop3, static_cast(INT32_MIN), static_cast(-1), + static_cast(INT32_MAX), static_cast(INT32_MIN)}, + TestCase{cdna5::kVAddMinI32Vop3, static_cast(INT32_MIN), static_cast(-1), + static_cast(INT32_MAX), static_cast(INT32_MIN), false}, + TestCase{cdna5::kVAddMinU32Vop3, UINT32_MAX, 1u, UINT32_MAX, UINT32_MAX}, + TestCase{cdna5::kVAddMinU32Vop3, UINT32_MAX, 1u, UINT32_MAX, UINT32_MAX, false}, + TestCase{cdna5::kVSadHiU8Vop3, UINT32_MAX, 0u, 0xfc040000u, UINT32_MAX}, + TestCase{cdna5::kVSadU8Vop3, 0x00010203u, 0x10002030u, UINT32_MAX, UINT32_MAX}, + TestCase{cdna5::kVSadU16Vop3, 0x00010003u, 0x10002000u, UINT32_MAX, UINT32_MAX}, + TestCase{cdna5::kVSadU32Vop3, 0u, UINT32_MAX, 1u, UINT32_MAX}, + TestCase{cdna5::kVMsadU8Vop3, 0xffffffffu, 0x01010101u, UINT32_MAX, UINT32_MAX}, + }; + + for (uint32_t scalar = 0; scalar < 2; ++scalar) { + util::set_force_scalar_for_testing(scalar != 0); + for (const TestCase &test : kCases) { + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + const uint32_t base = wf->vgpr_alloc().base; + cu->write_vgpr(base, 0, test.src0); + cu->write_vgpr(base + 1, 0, test.src1); + cu->write_vgpr(base + 2, 0, test.src2); + const auto words = cdna5::build_vop3(test.opcode, {.vdst = 3, + .clamp = static_cast(test.clamp), + .src0 = 256, + .src1 = 257, + .src2 = 258}); + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA5); + ASSERT_NE(decoder, nullptr); + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + inst->execute(*inst, wf); + EXPECT_EQ(cu->read_vgpr(base + 3, 0), test.expected) << "scalar " << scalar; + } + } +} + +TEST(Gfx1250ExecutionTest, Vop3U64ClampSaturatesNormalAndDppPaths) { + constexpr uint32_t kSrc0 = 0; + constexpr uint32_t kSrc1 = 2; + constexpr uint32_t kDst = 4; + + auto run_case = [](uint16_t opcode, uint64_t lhs, uint64_t rhs, uint64_t expected, bool dpp) { + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + const uint32_t base = wf->vgpr_alloc().base; + auto write_vgpr64 = [&](uint32_t reg, uint64_t value) { + cu->write_vgpr(base + reg, 0, static_cast(value)); + cu->write_vgpr(base + reg + 1, 0, static_cast(value >> 32)); + }; + auto read_vgpr64 = [&](uint32_t reg) { + return static_cast(cu->read_vgpr(base + reg, 0)) | + (static_cast(cu->read_vgpr(base + reg + 1, 0)) << 32); + }; + write_vgpr64(kSrc0, lhs); + write_vgpr64(kSrc1, rhs); + write_vgpr64(kDst, 0u); + + if (!dpp) { + const auto words = cdna5::build_vop3( + opcode, {.vdst = kDst, .clamp = 1, .src0 = 256 + kSrc0, .src1 = 256 + kSrc1}); + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA5); + ASSERT_NE(decoder, nullptr); + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + inst->execute(*inst, wf); + } else { + cdna5::Vop3VopDpp16MachineInst raw{}; + raw.vdst = kDst; + raw.clamp = 1; + raw.src0 = amdgpu::SRC_DPP; + raw.src1 = 256 + kSrc1; + raw.vsrc0 = kSrc0; + raw.dpp_ctrl = amdgpu::dpp::ROW_SELECT_BASE; + raw.bound_ctrl = 1; + raw.bank_mask = 0xF; + raw.row_mask = 0xF; + if (opcode == cdna5::kVAddNcU64Vop3) { + cdna5::VAddNcU64Vop3 inst(reinterpret_cast(&raw)); + inst.execute_impl(*wf); + } else { + ASSERT_EQ(opcode, cdna5::kVSubNcU64Vop3); + cdna5::VSubNcU64Vop3 inst(reinterpret_cast(&raw)); + inst.execute_impl(*wf); + } + } + + EXPECT_EQ(read_vgpr64(kDst), expected) << "dpp " << dpp; + }; + + for (bool dpp : {false, true}) { + run_case(cdna5::kVAddNcU64Vop3, UINT64_MAX, 1u, UINT64_MAX, dpp); + run_case(cdna5::kVSubNcU64Vop3, 0u, 1u, 0u, dpp); + } +} + +TEST(Gfx1250ExecutionTest, Vop3Mad64ClampSaturatesAndPreservesCarryNormalAndDpp) { + struct TestCase { + bool is_signed; + bool writes_carry; + uint32_t src0; + uint32_t src1; + uint64_t src2; + uint64_t expected; + bool expected_carry; + }; + constexpr std::array kCases{ + TestCase{false, false, UINT32_MAX, UINT32_MAX, UINT64_MAX, UINT64_MAX, false}, + TestCase{false, true, UINT32_MAX, UINT32_MAX, UINT64_MAX, UINT64_MAX, true}, + TestCase{false, true, 2u, 3u, 4u, 10u, false}, + TestCase{true, false, static_cast(INT32_MAX), static_cast(INT32_MAX), + static_cast(INT64_MAX), static_cast(INT64_MAX), false}, + TestCase{true, false, static_cast(INT32_MIN), static_cast(INT32_MAX), + static_cast(INT64_MIN), static_cast(INT64_MIN), false}, + TestCase{true, true, static_cast(INT32_MAX), static_cast(INT32_MAX), + static_cast(INT64_MAX), static_cast(INT64_MAX), true}, + TestCase{true, true, static_cast(INT32_MIN), static_cast(INT32_MAX), + static_cast(INT64_MIN), static_cast(INT64_MIN), true}, + TestCase{true, true, static_cast(-2), 3u, 10u, 4u, false}, + }; + + for (bool dpp : {false, true}) { + for (const TestCase &test : kCases) { + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + const uint32_t base = wf->vgpr_alloc().base; + cu->write_vgpr(base, 0, test.src0); + cu->write_vgpr(base + 1, 0, test.src1); + cu->write_vgpr(base + 2, 0, static_cast(test.src2)); + cu->write_vgpr(base + 3, 0, static_cast(test.src2 >> 32)); + cu->write_vgpr(base + 4, 0, 0u); + cu->write_vgpr(base + 5, 0, 0u); + // Seed CO opposite to the expected bit so every case proves it is replaced. + write_wave_sgpr(*cu, *wf, 2, + test.writes_carry ? static_cast(!test.expected_carry) : 0u); + + if (!dpp) { + std::array words; + if (test.writes_carry) { + const uint16_t opcode = + test.is_signed ? cdna5::kVMadCoI64I32Vop3SdstEnc : cdna5::kVMadCoU64U32Vop3SdstEnc; + words = cdna5::build_vop3_sdst_enc( + opcode, {.vdst = 4, .sdst = 2, .clamp = 1, .src0 = 256, .src1 = 257, .src2 = 258}); + } else { + const uint16_t opcode = + test.is_signed ? cdna5::kVMadNcI64I32Vop3 : cdna5::kVMadNcU64U32Vop3; + words = cdna5::build_vop3(opcode, + {.vdst = 4, .clamp = 1, .src0 = 256, .src1 = 257, .src2 = 258}); + } + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA5); + ASSERT_NE(decoder, nullptr); + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + inst->execute(*inst, wf); + } else if (test.writes_carry) { + cdna5::Vop3SdstEncVopDpp16MachineInst raw{}; + raw.vdst = 4; + raw.sdst = 2; + raw.clamp = 1; + raw.src0 = amdgpu::SRC_DPP; + raw.src1 = 257; + raw.src2 = 258; + raw.vsrc0 = 0; + raw.dpp_ctrl = amdgpu::dpp::ROW_SELECT_BASE; + raw.bound_ctrl = 1; + raw.bank_mask = 0xF; + raw.row_mask = 0xF; + if (test.is_signed) { + cdna5::VMadCoI64I32Vop3SdstEnc inst(reinterpret_cast(&raw)); + inst.execute_impl(*wf); + } else { + cdna5::VMadCoU64U32Vop3SdstEnc inst(reinterpret_cast(&raw)); + inst.execute_impl(*wf); + } + } else { + cdna5::Vop3VopDpp16MachineInst raw{}; + raw.vdst = 4; + raw.clamp = 1; + raw.src0 = amdgpu::SRC_DPP; + raw.src1 = 257; + raw.src2 = 258; + raw.vsrc0 = 0; + raw.dpp_ctrl = amdgpu::dpp::ROW_SELECT_BASE; + raw.bound_ctrl = 1; + raw.bank_mask = 0xF; + raw.row_mask = 0xF; + if (test.is_signed) { + cdna5::VMadNcI64I32Vop3 inst(reinterpret_cast(&raw)); + inst.execute_impl(*wf); + } else { + cdna5::VMadNcU64U32Vop3 inst(reinterpret_cast(&raw)); + inst.execute_impl(*wf); + } + } + + const uint64_t result = static_cast(cu->read_vgpr(base + 4, 0)) | + (static_cast(cu->read_vgpr(base + 5, 0)) << 32); + EXPECT_EQ(result, test.expected) + << "signed " << test.is_signed << " co " << test.writes_carry << " dpp " << dpp; + if (test.writes_carry) { + EXPECT_EQ(read_wave_sgpr(*cu, *wf, 2) & 1u, test.expected_carry); + } + } + } + + // CLAMP=0 keeps the modulo destination while CO still reports overflow. + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + const uint32_t base = wf->vgpr_alloc().base; + cu->write_vgpr(base, 0, UINT32_MAX); + cu->write_vgpr(base + 1, 0, UINT32_MAX); + cu->write_vgpr(base + 2, 0, UINT32_MAX); + cu->write_vgpr(base + 3, 0, UINT32_MAX); + const auto words = + cdna5::build_vop3_sdst_enc(cdna5::kVMadCoU64U32Vop3SdstEnc, + {.vdst = 4, .sdst = 2, .src0 = 256, .src1 = 257, .src2 = 258}); + cdna5::VMadCoU64U32Vop3SdstEnc inst(words.data()); + inst.execute_impl(*wf); + const uint64_t result = static_cast(cu->read_vgpr(base + 4, 0)) | + (static_cast(cu->read_vgpr(base + 5, 0)) << 32); + EXPECT_EQ(result, 0xfffffffe00000000ULL); + EXPECT_EQ(read_wave_sgpr(*cu, *wf, 2) & 1u, 1u); +} + +TEST(Gfx1250ExecutionTest, Vop3PackedIntegerClampSaturatesSelectedHalves) { + ForceScalarGuard guard; + struct TestCase { + uint16_t opcode; + uint32_t src0; + uint32_t src1; + uint32_t expected; + }; + constexpr std::array kCases{ + TestCase{cdna5::kVPkAddI16Vop3p, 0x7fff8000u, 0x0001ffffu, 0x80007fffu}, + TestCase{cdna5::kVPkSubI16Vop3p, 0x7fff8000u, 0xffff0001u, 0x80007fffu}, + TestCase{cdna5::kVPkAddU16Vop3p, 0xffff0000u, 0x00010001u, 0x0001ffffu}, + TestCase{cdna5::kVPkSubU16Vop3p, 0x0000ffffu, 0x00010001u, 0xfffe0000u}, + }; + for (uint32_t scalar = 0; scalar < 2; ++scalar) { + util::set_force_scalar_for_testing(scalar != 0); + for (const TestCase &test : kCases) { + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + const uint32_t base = wf->vgpr_alloc().base; + cu->write_vgpr(base, 0, test.src0); + cu->write_vgpr(base + 1, 0, test.src1); + const auto words = cdna5::build_vop3p( + test.opcode, + {.vdst = 2, .opsel = 3, .clamp = 1, .src0 = 256, .src1 = 257, .opsel_hi = 0}); + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA5); + ASSERT_NE(decoder, nullptr); + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + inst->execute(*inst, wf); + EXPECT_EQ(cu->read_vgpr(base + 2, 0), test.expected) << "scalar " << scalar; + } + } +} + +TEST(Gfx1250ExecutionTest, Vop3PackedMadClampSaturatesSelectedHalves) { + ForceScalarGuard guard; + struct TestCase { + uint16_t opcode; + uint32_t src0; + uint32_t src1; + uint32_t src2; + uint32_t expected; + uint8_t opsel; + uint8_t opsel_hi; + uint8_t opsel_hi_2; + bool clamp = true; + }; + constexpr std::array kCases{ + // Default selectors exercise the SIMD path when CLAMP is clear. + TestCase{cdna5::kVPkMadI16Vop3p, 0x80007fffu, 0x00020002u, 0x00000000u, 0x80007fffu, 0u, 3u, + 1u}, + TestCase{cdna5::kVPkMadU16Vop3p, 0x0001ffffu, 0x00020002u, 0x00000000u, 0x0002ffffu, 0u, 3u, + 1u}, + TestCase{cdna5::kVPkMadI16Vop3p, 0x80007fffu, 0x00020002u, 0x00000000u, 0x0000fffeu, 0u, 3u, + 1u, false}, + TestCase{cdna5::kVPkMadU16Vop3p, 0x0001ffffu, 0x00020002u, 0x00000000u, 0x0002fffeu, 0u, 3u, + 1u, false}, + // Non-default selectors force scalar fallback and verify half selection. + TestCase{cdna5::kVPkMadI16Vop3p, 0x7fff8000u, 0x00020002u, 0x00000000u, 0x80007fffu, 7u, 0u, + 0u}, + TestCase{cdna5::kVPkMadU16Vop3p, 0xffff0001u, 0x00020002u, 0x00010000u, 0x0002ffffu, 3u, 0u, + 0u}, + TestCase{cdna5::kVPkMadI16Vop3p, 0x7fff8000u, 0x00020002u, 0x00000000u, 0x0000fffeu, 7u, 0u, + 0u, false}, + TestCase{cdna5::kVPkMadU16Vop3p, 0xffff0001u, 0x00020002u, 0x00010000u, 0x0002fffeu, 3u, 0u, + 0u, false}, + }; + for (uint32_t scalar = 0; scalar < 2; ++scalar) { + util::set_force_scalar_for_testing(scalar != 0); + for (const TestCase &test : kCases) { + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + const uint32_t base = wf->vgpr_alloc().base; + cu->write_vgpr(base, 0, test.src0); + cu->write_vgpr(base + 1, 0, test.src1); + cu->write_vgpr(base + 2, 0, test.src2); + const auto words = cdna5::build_vop3p(test.opcode, {.vdst = 3, + .opsel = test.opsel, + .opsel_hi_2 = test.opsel_hi_2, + .clamp = static_cast(test.clamp), + .src0 = 256, + .src1 = 257, + .src2 = 258, + .opsel_hi = test.opsel_hi}); + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA5); + ASSERT_NE(decoder, nullptr); + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + inst->execute(*inst, wf); + EXPECT_EQ(cu->read_vgpr(base + 3, 0), test.expected) << "scalar " << scalar; + } + } +} + +TEST(Gfx1250ExecutionTest, Vop3CarryClampSaturatesVdstAndPreservesCarryBorrow) { + ForceScalarGuard guard; + struct TestCase { + uint16_t opcode; + uint32_t src0; + uint32_t src1; + bool carry_in; + uint32_t expected_vdst; + bool expected_carry; + bool initial_carry = false; + }; + constexpr std::array kCases{ + TestCase{cdna5::kVAddCoU32Vop3SdstEnc, UINT32_MAX, 1u, false, UINT32_MAX, true}, + TestCase{cdna5::kVSubCoU32Vop3SdstEnc, 0u, 1u, false, 0u, true}, + TestCase{cdna5::kVSubrevCoU32Vop3SdstEnc, 1u, 0u, false, 0u, true}, + TestCase{cdna5::kVAddCoCiU32Vop3SdstEnc, UINT32_MAX, 0u, true, UINT32_MAX, true}, + TestCase{cdna5::kVSubCoCiU32Vop3SdstEnc, 0u, 0u, true, 0u, true}, + TestCase{cdna5::kVSubrevCoCiU32Vop3SdstEnc, 0u, 0u, true, 0u, true}, + TestCase{cdna5::kVAddCoU32Vop3SdstEnc, 1u, 2u, false, 3u, false, true}, + TestCase{cdna5::kVSubCoU32Vop3SdstEnc, 2u, 1u, false, 1u, false, true}, + }; + for (uint32_t scalar = 0; scalar < 2; ++scalar) { + util::set_force_scalar_for_testing(scalar != 0); + for (const TestCase &test : kCases) { + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + const uint32_t base = wf->vgpr_alloc().base; + cu->write_vgpr(base, 0, test.src0); + cu->write_vgpr(base + 1, 0, test.src1); + write_wave_sgpr(*cu, *wf, 4, test.carry_in ? 1u : 0u); + write_wave_sgpr(*cu, *wf, 2, test.initial_carry ? 1u : 0u); + const auto words = cdna5::build_vop3_sdst_enc( + test.opcode, {.vdst = 2, .sdst = 2, .clamp = 1, .src0 = 256, .src1 = 257, .src2 = 4}); + auto decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA5); + ASSERT_NE(decoder, nullptr); + std::unique_ptr inst(decode_valid(*decoder, words.data())); + ASSERT_NE(inst, nullptr); + inst->execute(*inst, wf); + EXPECT_EQ(cu->read_vgpr(base + 2, 0), test.expected_vdst) << "scalar " << scalar; + EXPECT_EQ(read_wave_sgpr(*cu, *wf, 2) & 1u, test.expected_carry ? 1u : 0u) + << "scalar " << scalar; + } + } +} + TEST(FpModePolicyTest, F64HelpersRestoreAmbientHostEnvironment) { HostFenvGuard environment_guard; ASSERT_EQ(std::fesetround(FE_DOWNWARD), 0); @@ -2779,6 +3277,39 @@ TEST(Gfx1250ExecutionTest, QsadAndMqsadUseFourSlidingWindowsAndMaskedReferenceBy EXPECT_EQ(read(6), 202u); EXPECT_EQ(read(7), 304u); EXPECT_EQ(read(8), 406u); + + // CLAMP saturates each packed or full-width accumulation independently. + write(0, 0u); + write(1, 0u); + write(2, 0xffff'ffffu); + write(4, 0xffff'ffffu); + write(5, 0xffff'ffffu); + const auto clamped_qsad_words = cdna5::build_vop3( + cdna5::kVQsadPkU16U8Vop3, {.vdst = 8, .clamp = 1, .src0 = 256, .src1 = 258, .src2 = 260}); + cdna5::VQsadPkU16U8Vop3 clamped_qsad(clamped_qsad_words.data()); + clamped_qsad.execute_impl(*wf); + EXPECT_EQ(read(8), UINT32_MAX); + EXPECT_EQ(read(9), UINT32_MAX); + + const auto clamped_mqsad_pk_words = cdna5::build_vop3( + cdna5::kVMqsadPkU16U8Vop3, {.vdst = 8, .clamp = 1, .src0 = 256, .src1 = 258, .src2 = 260}); + cdna5::VMqsadPkU16U8Vop3 clamped_mqsad_pk(clamped_mqsad_pk_words.data()); + clamped_mqsad_pk.execute_impl(*wf); + EXPECT_EQ(read(8), UINT32_MAX); + EXPECT_EQ(read(9), UINT32_MAX); + + write(4, UINT32_MAX); + write(5, UINT32_MAX); + write(6, UINT32_MAX); + write(7, UINT32_MAX); + const auto clamped_mqsad_words = cdna5::build_vop3( + cdna5::kVMqsadU32U8Vop3, {.vdst = 12, .clamp = 1, .src0 = 256, .src1 = 258, .src2 = 260}); + cdna5::VMqsadU32U8Vop3 clamped_mqsad(clamped_mqsad_words.data()); + clamped_mqsad.execute_impl(*wf); + EXPECT_EQ(read(12), UINT32_MAX); + EXPECT_EQ(read(13), UINT32_MAX); + EXPECT_EQ(read(14), UINT32_MAX); + EXPECT_EQ(read(15), UINT32_MAX); } TEST(Gfx1250ExecutionTest, MullitAndTrigPreopPreserveLegacyAndRangeReductionRules) { diff --git a/emulation/rocjitsu/tests/simd_correctness/vop3_shift64_simd_correctness_test.cpp b/emulation/rocjitsu/tests/simd_correctness/vop3_shift64_simd_correctness_test.cpp index c5bc04c3d37b..301472be9f46 100644 --- a/emulation/rocjitsu/tests/simd_correctness/vop3_shift64_simd_correctness_test.cpp +++ b/emulation/rocjitsu/tests/simd_correctness/vop3_shift64_simd_correctness_test.cpp @@ -60,8 +60,10 @@ constexpr void vop3_encode(uint32_t op, uint32_t vdst, uint32_t src0, uint32_t s // VOP3_SDST_ENC: word0 also carries sdst[14:8]. constexpr void vop3_sdstenc_encode(uint32_t op, uint32_t vdst, uint32_t sdst, uint32_t src0, - uint32_t src1, uint32_t src2, uint32_t words[2]) { - words[0] = (vdst & 0xFF) | ((sdst & 0x7F) << 8) | ((op & 0x3FF) << 16) | (0x34u << 26); + uint32_t src1, uint32_t src2, uint32_t words[2], + uint32_t clamp = 0) { + words[0] = (vdst & 0xFF) | ((sdst & 0x7F) << 8) | ((clamp & 0x1) << 15) | ((op & 0x3FF) << 16) | + (0x34u << 26); words[1] = (src0 & 0x1FF) | ((src1 & 0x1FF) << 9) | ((src2 & 0x1FF) << 18); } @@ -316,6 +318,52 @@ TEST(Vop3Shift64SimdCorrectness, MadWide64) { check_mad64("v_mad_i64_i32_vop3", 489, /*is_signed=*/true, 0xA5A5'F0F0'1234'8001ULL); } +TEST(Vop3Shift64SimdCorrectness, MadWide64ClampSaturatesAndPreservesCarry) { + ForceScalarGuard gate_guard; + struct ClampCase { + bool is_signed; + uint32_t src0; + uint32_t src1; + uint64_t src2; + uint64_t expected; + }; + constexpr std::array kCases{ + ClampCase{false, UINT32_MAX, UINT32_MAX, UINT64_MAX, UINT64_MAX}, + ClampCase{true, static_cast(INT32_MIN), static_cast(INT32_MAX), + static_cast(INT64_MIN), static_cast(INT64_MIN)}, + ClampCase{true, static_cast(INT32_MAX), static_cast(INT32_MAX), + static_cast(INT64_MAX), static_cast(INT64_MAX)}, + }; + for (const ClampCase &test : kCases) { + for (bool force_scalar : {false, true}) { + util::set_force_scalar_for_testing(force_scalar); + Fixture fx; + ASSERT_NE(fx.cu, nullptr); + ASSERT_NE(fx.wf, nullptr); + const uint32_t sb = fx.wf->sgpr_alloc().base; + ASSERT_EQ(sb % 2u, 0u); + const uint32_t opcode = test.is_signed ? 489u : 488u; + uint32_t words[2] = {0u, 0u}; + vop3_sdstenc_encode(opcode, kDstVgpr, sb, /*src0=*/256, /*src1=*/258, /*src2=*/260, words, + /*clamp=*/1); + std::unique_ptr inst(decode_valid(*fx.decoder, words)); + ASSERT_NE(inst, nullptr); + const uint32_t vb = fx.wf->vgpr_alloc().base; + fx.cu->write_vgpr(vb, 0, test.src0); + fx.cu->write_vgpr(vb + 2, 0, test.src1); + fx.write64(vb + 4, 0, test.src2); + fx.write64(vb + kDstVgpr, 0, 0u); + fx.write_sgpr64(sb, 0u); + fx.wf->set_exec(1u); + fx.cu->execute_instruction(inst.get(), *fx.wf); + EXPECT_EQ(fx.read64(vb + kDstVgpr, 0), test.expected) + << "signed " << test.is_signed << " scalar " << force_scalar; + EXPECT_EQ(fx.read_sgpr64(sb) & 1u, 1u) + << "signed " << test.is_signed << " scalar " << force_scalar; + } + } +} + TEST(Vop3Shift64SimdCorrectness, RevShiftsFullExec) { if constexpr (!util::has_stdx_simd) { GTEST_SKIP() << " unavailable — scalar fallback in use"; diff --git a/emulation/rocjitsu/tests/simd_correctness/vop3_ternary_int_simd_correctness_test.cpp b/emulation/rocjitsu/tests/simd_correctness/vop3_ternary_int_simd_correctness_test.cpp index 62652d917517..0ff72c05b6b4 100644 --- a/emulation/rocjitsu/tests/simd_correctness/vop3_ternary_int_simd_correctness_test.cpp +++ b/emulation/rocjitsu/tests/simd_correctness/vop3_ternary_int_simd_correctness_test.cpp @@ -16,6 +16,10 @@ #include "util/simd_test_hooks.h" #include "rocjitsu/code/rj_code.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna4/builders.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna4/execution_backend.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna4/opcodes.h" +#include "rocjitsu/isa/arch/amdgpu/generated/rdna4/vop3.h" #include "rocjitsu/isa/arch/amdgpu/generated/shared/execute_shared.h" #include "rocjitsu/isa/decoder.h" #include "rocjitsu/isa/instruction.h" @@ -116,15 +120,16 @@ struct Fixture { std::unique_ptr decoder; amdgpu::Wavefront *wf = nullptr; - Fixture() : gpu_mem("vop3_tern_int_mem"), l2("vop3_tern_int_l2") { + explicit Fixture(rj_code_arch_t arch = ROCJITSU_CODE_ARCH_CDNA4) + : gpu_mem("vop3_tern_int_mem"), l2("vop3_tern_int_l2") { amdgpu::ComputeUnitCore::Config cfg{}; - cfg.arch = ROCJITSU_CODE_ARCH_CDNA4; + cfg.arch = arch; cfg.num_wf_slots = 1; cfg.sgprs_per_wf = SGPRS_PER_WF; cfg.vgprs_per_wf = VGPRS_PER_WF; cfg.lds_size_kb = 64; cu = amdgpu::ComputeUnitCore::create("cu_vop3_tern_int", cfg, &gpu_mem, &l2); - decoder = Decoder::create(ROCJITSU_CODE_ARCH_CDNA4); + decoder = Decoder::create(arch); wf = cu->dispatch_wf(0, 0, SGPRS_PER_WF, VGPRS_PER_WF); }