[rocjitsu] Fix gfx1250 fpsan numerical failures

emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py

@@ -94,6 +94,8 @@ def _register_handlers() -> None:
         gen_vector_permlane_swap,
         gen_vector_permlane,
         gen_vector_permlane64,
+        gen_vector_permlane_family,
+        gen_vector_permlane_idx_gen,
         gen_vector_cvt_pk,
         gen_vector_cvt_scale,
         gen_cvt_fp8,
@@ -169,7 +171,14 @@ def _register_handlers() -> None:
         c.dst_ops, c.src_ops
     )
     DISPATCH['vector_bitop3'] = lambda c: gen_vector_bitop3(
-        c.dst_ops, c.src_ops, c.dtype
+        c.dst_ops,
+        c.src_ops,
+        c.dtype,
+        (
+            c.opsel_exprs[0]
+            if c.arch_name == 'gfx1250' and c.is_vop3 and c.dtype == 'b16'
+            else None
+        ),
     )
     DISPATCH['vector_permlane16'] = lambda c: gen_vector_permlane(
         c.dst_ops, c.src_ops, c.op, cross=False, op_sel_expr=c.opsel_exprs[0]
@@ -186,6 +195,12 @@ def _register_handlers() -> None:
     DISPATCH['vector_permlane64'] = lambda c: gen_vector_permlane64(
         c.dst_ops, c.src_ops
     )
+    DISPATCH['vector_permlane_family'] = lambda c: gen_vector_permlane_family(
+        c.dst_ops, c.src_ops, c.op
+    )
+    DISPATCH['vector_permlane_idx_gen'] = lambda c: gen_vector_permlane_idx_gen(
+        c.dst_ops, c.src_ops
+    )
     DISPATCH['vector_cvt_pk'] = lambda c: gen_vector_cvt_pk(
         c.dst_ops,
         c.src_ops,
@@ -196,6 +211,14 @@ def _register_handlers() -> None:
             if c.is_vop3 and 'opsel' in c.enc_field_names
             else 'inst_.op_sel' if c.is_vop3 and 'op_sel' in c.enc_field_names else '0u'
         ),
+        fp8_format_select=(
+            'inst_.clamp'
+            if c.cls == 'vector_cvt_pk'
+            and c.op in ('fp8_f32', 'fp8_f16')
+            and c.is_vop3
+            and c.arch_name == 'gfx1250'
+            else None
+        ),
     )
     DISPATCH['vector_cvt_scale'] = lambda c: gen_vector_cvt_scale(
         c.dst_ops, c.src_ops, c.cls, c.op

emulation/rocjitsu/lib/python/amdisa/codegen/execute/matrix.py

@@ -58,6 +58,10 @@ def _gfx1250_wmma_spec(
         if input_type == 'BF16' and (M, N, K) == (16, 16, 32):
             return f'exec_wmma_bf16_spec<{M}, {N}, {K}>'
         return None
+    if result_type == 'BF16F32':
+        if input_type == 'BF16' and (M, N, K) == (16, 16, 32):
+            return 'exec_wmma_bf16f32_16x16x32_bf16'
+        return None
     # result F32
     if input_type in ('F32', 'XF32') and N % 16 == 0:
         return f'exec_wmma_f32_f32_spec<{M}, {N}, {K}>'
@@ -124,9 +128,7 @@ def gen_mfma(
             f'  throw util::UnimplementedInst(mnemonic());'
         )
 
-    result_type = m.group(1)  # F32, I32, F64
-    if result_type == 'BF16F32':
-        result_type = 'F32'
+    result_type = m.group(1)  # F32, I32, F64, BF16F32
     M, N, K = int(m.group(2)), int(m.group(3)), int(m.group(4))
     input_type = m.group(5)  # F32, XF32, F16, BF16, I8, F64, etc.
     is_swmmac = name.startswith('V_SWMMAC_')
@@ -395,7 +397,10 @@ def gen_mfma(
             )
             L.append(f'            src0_base,')
             L.append(f'            src1_base,')
-            L.append(f'            s2, extract_a, extract_b, const_acc);')
+            L.append(
+                f'            s2, extract_a, extract_b, const_acc,'
+                f' amdgpu::wmma_c_modifier(inst_.neg, inst_.neg_hi));'
+            )
             L.append(f'      }});')
             L.append(f'  if (!dispatched)')
             L.append(f'    throw util::UnimplementedInst(mnemonic());')
@@ -424,20 +429,33 @@ def gen_mfma(
                 # the generic exec_wmma_* runtime path.
                 spec = _gfx1250_wmma_spec(result_type, input_type, M, N, K)
                 if spec is not None:
-                    L.append(
-                        f'  amdgpu::{spec}(cu, dst, src0_base, src1_base, s2, const_acc);'
-                    )
+                    if result_type in ('F32', 'BF16F32'):
+                        L.append(
+                            f'  amdgpu::{spec}(cu, dst, src0_base, src1_base, s2, const_acc,'
+                            f' amdgpu::wmma_c_modifier(inst_.neg, inst_.neg_hi));'
+                        )
+                    else:
+                        L.append(
+                            f'  amdgpu::{spec}(cu, dst, src0_base, src1_base, s2, const_acc);'
+                        )
                 else:
                     if result_type == 'F16':
                         exec_fn = 'exec_wmma_f16'
                     elif result_type == 'BF16':
                         exec_fn = 'exec_wmma_bf16'
                     else:
                         exec_fn = 'exec_wmma_f32'
-                    L.append(
-                        f'  amdgpu::{exec_fn}(cu, {M}, {N}, {K}, {in_bits}, dst, src0_base,'
-                        f' src1_base, s2, {ea}, {eb}, const_acc);'
-                    )
+                    if exec_fn == 'exec_wmma_f32':
+                        L.append(
+                            f'  amdgpu::{exec_fn}(cu, {M}, {N}, {K}, {in_bits}, dst, src0_base,'
+                            f' src1_base, s2, {ea}, {eb}, const_acc,'
+                            f' amdgpu::wmma_c_modifier(inst_.neg, inst_.neg_hi));'
+                        )
+                    else:
+                        L.append(
+                            f'  amdgpu::{exec_fn}(cu, {M}, {N}, {K}, {in_bits}, dst, src0_base,'
+                            f' src1_base, s2, {ea}, {eb}, const_acc);'
+                        )
         elif input_type in ('F8_F6_F4', 'F8F6F4'):
             # f8f6f4 MFMA: cbsz/blgp encode data format, NOT lane
             # permutation. Use dispatch_matrix_fmt_pair to select the

emulation/rocjitsu/lib/python/amdisa/codegen/execute/sema_lower.py

@@ -120,6 +120,8 @@ class LoweringContext:
     true16_src_selects: dict[int, str] = field(default_factory=dict)
     true16_dst_reg: str | None = None
     true16_src_raw: str | None = None
+    fp8_byte_select: str | None = None
+    fp8_decode_e5m3_select: str | None = None
     vector_sgpr_once: bool = False
     clear_false_lane_mask_writes: bool = True
 
@@ -321,6 +323,8 @@ def _lower_stmt(node: SemaNode, ctx: LoweringContext) -> list[str]:
             true16_src_selects=ctx.true16_src_selects,
             true16_dst_reg=ctx.true16_dst_reg,
             true16_src_raw=ctx.true16_src_raw,
+            fp8_byte_select=ctx.fp8_byte_select,
+            fp8_decode_e5m3_select=ctx.fp8_decode_e5m3_select,
             vector_sgpr_once=ctx.vector_sgpr_once,
             clear_false_lane_mask_writes=ctx.clear_false_lane_mask_writes,
         )
@@ -453,6 +457,8 @@ def _lower_if(node: SemaNode, ctx: LoweringContext) -> list[str]:
         true16_src_selects=ctx.true16_src_selects,
         true16_dst_reg=ctx.true16_dst_reg,
         true16_src_raw=ctx.true16_src_raw,
+        fp8_byte_select=ctx.fp8_byte_select,
+        fp8_decode_e5m3_select=ctx.fp8_decode_e5m3_select,
         vector_sgpr_once=ctx.vector_sgpr_once,
         clear_false_lane_mask_writes=ctx.clear_false_lane_mask_writes,
     )
@@ -504,6 +510,8 @@ def _lower_for(node: SemaNode, ctx: LoweringContext) -> list[str]:
         true16_src_selects=ctx.true16_src_selects,
         true16_dst_reg=ctx.true16_dst_reg,
         true16_src_raw=ctx.true16_src_raw,
+        fp8_byte_select=ctx.fp8_byte_select,
+        fp8_decode_e5m3_select=ctx.fp8_decode_e5m3_select,
         vector_sgpr_once=ctx.vector_sgpr_once,
         clear_false_lane_mask_writes=ctx.clear_false_lane_mask_writes,
     )
@@ -1051,6 +1059,14 @@ def _lower_dst_write(
                 dst_ref = f'wf.vgpr_alloc().base + ({ctx.true16_dst_reg})'
                 read_dst = f'wf.cu().read_vgpr({dst_ref}, lane)'
                 write_dst = f'wf.cu().write_vgpr({dst_ref}, lane, merged);'
+            elif ctx.true16_dst_select == 'inst_.opsel & 0x8u':
+                return [
+                    f'{ind}{{',
+                    f'{ind}  uint32_t src_half = static_cast<uint32_t>(static_cast<uint16_t>({selected_rhs}));',
+                    f'{ind}  ::rocjitsu::amdgpu::write_vop3_true16_dst('
+                    f'{name}, wf, lane, {ctx.true16_dst_select}, src_half);',
+                    f'{ind}}}',
+                ]
             else:
                 read_dst = f'{name}.read_lane(wf, lane)'
                 write_dst = f'{name}.write_lane(wf, lane, merged);'
@@ -1681,6 +1697,29 @@ def _lower_call(node: SemaNode, ctx: LoweringContext) -> str:
     args = [_lower_expr(c, ctx) for c in node.children[1:]]
     args_str = ', '.join(args)
 
+    if len(args) == 1 and callee in (
+        'cvt_f32_fp8',
+        'cvt_f32_bf8',
+        'cvt_f16_fp8',
+        'cvt_f16_bf8',
+    ):
+        arg = args[0]
+        if ctx.fp8_byte_select is not None:
+            arg = f'(({arg} >> (({ctx.fp8_byte_select}) * 8u)) & 0xFFu)'
+        if ctx.fp8_decode_e5m3_select is not None and callee == 'cvt_f32_fp8':
+            return (
+                f'std::bit_cast<uint32_t>(({ctx.fp8_decode_e5m3_select}) ? '
+                f'util::fp8_e5m3_to_f32(static_cast<uint8_t>({arg})) : '
+                f'util::fp8_e4m3_to_f32(static_cast<uint8_t>({arg})))'
+            )
+        if ctx.fp8_decode_e5m3_select is not None and callee == 'cvt_f16_fp8':
+            return (
+                f'static_cast<uint32_t>(util::f32_to_f16(({ctx.fp8_decode_e5m3_select}) ? '
+                f'util::fp8_e5m3_to_f32(static_cast<uint8_t>({arg})) : '
+                f'util::fp8_e4m3_to_f32(static_cast<uint8_t>({arg}))))'
+            )
+        return _INLINE_UNARY_OPS[callee].format(arg)
+
     if len(args) == 1 and callee in _INLINE_UNARY_OPS:
         return _INLINE_UNARY_OPS[callee].format(args[0])
     if len(args) == 2 and callee in _INLINE_BINARY_OPS:

emulation/rocjitsu/lib/python/amdisa/codegen/execute/simd_codegen.py

@@ -336,8 +336,10 @@
 # widens it to f32, applies omod/clamp, then narrows back to u16 (a 16-bit->f32->16-bit
 # modifier round trip). The plain `a & 0xFFFFu` VOP1 functor it would otherwise reuse
 # ignores those modifiers, so with clamp/omod set the SIMD path diverged from scalar
-# (clamp=1: scalar 0x1 vs simd 0xffff). Leaving the VOP3 twin scalar keeps it correct;
-# the modifier-free VOP1 form still takes the fast path.
+# (clamp=1: scalar 0x1 vs simd 0xffff). The f32 FP8/BF8 VOP3 decoders use op_sel[1:0]
+# as a bit-swapped byte selector, while the VOP1 SIMD functor always consumes byte 0. Leaving these
+# VOP3 twins scalar keeps both modifier and byte-select behavior correct; the
+# modifier-free / byte0 VOP1 forms still take the fast path.
 _VOP3_UNARY_SKIP = {
     'v_floor_f16',
     'v_ceil_f16',
@@ -354,6 +356,8 @@
     'v_frexp_exp_i32_f64',
     'v_frexp_mant_f16',
     'v_frexp_exp_i16_f16',
+    'v_cvt_f32_fp8',
+    'v_cvt_f32_bf8',
 }
 
 SIMD_VOP1_UNARY: dict[str, tuple[str, str, str]] = {
@@ -976,7 +980,7 @@
         'double',
         '[](auto a) { return util::native<double>(1.0) / util::stdx::sqrt(a); }',
     ),
-    'v_sqrt_f64_vop1': ('double', '[](auto a) { return util::stdx::sqrt(a); }'),
+    'v_sqrt_f64_vop1': ('double', '[](auto a) { return util::sqrt_f64_simd(a); }'),
     'v_mov_b64_vop1': ('uint64_t', '[](auto a) { return a; }'),
     # frexp mantissa: significand in [0.5,1) via bitfield rebias + denormal
     # renorm (see util::frexp_mant_f64_simd). VOP3 twin in SIMD_VOP3_UNARY_FP64.
@@ -1011,18 +1015,13 @@
     'v_cvt_i32_f64_vop1': (
         'int32_t',
         '[](auto s) {'
-        ' auto r = s;'
-        ' util::stdx::where(util::stdx::isnan(s), r) = 0.0;'
-        ' util::stdx::where(s >= 2147483648.0, r) = 2147483647.0;'
-        ' util::stdx::where(s < -2147483648.0, r) = -2147483648.0;'
+        ' auto r = util::cvt_i32_f64_saturate_input_simd(s);'
         ' return util::stdx::static_simd_cast<util::narrow32<int32_t>>(r); }',
     ),
     'v_cvt_u32_f64_vop1': (
         'uint32_t',
         '[](auto s) {'
-        ' auto r = s;'
-        ' util::stdx::where(util::stdx::isnan(s) || s < 0.0, r) = 0.0;'
-        ' util::stdx::where(s >= 4294967296.0, r) = 4294967295.0;'
+        ' auto r = util::cvt_u32_f64_saturate_input_simd(s);'
         ' return util::stdx::static_simd_cast<util::narrow32<uint32_t>>(r); }',
     ),
     # frexp exponent (f64 -> int32). util::frexp_exp_f64_simd returns the int32 in
@@ -1546,15 +1545,16 @@ def _build_simd_vopc64() -> dict[str, tuple[str, str]]:
 # v_cmp_class_f64: a 64-bit f64 value (src0) tested against a 32-bit class mask
 # (vsrc1), so it needs the mixed-width class glue (try_execute_vopc_class_f64_simd)
 # rather than the equal-width VOPC path. The functor receives src0 as
-# native<uint64_t> raw bits and vsrc1 as a native_width64-wide narrow32<uint32_t>
+# a uint64 SIMD raw-bit vector and vsrc1 as the matching uint32 SIMD class-mask
 # mask; it classifies the f64 from its raw bits (qNaN bit = mantissa MSB
 # 0x0008000000000000), partitions into one of the 10 mutually exclusive classes,
 # casts the small class code down to the 32-bit mask width, and returns
 # (cls & mask) != 0. Pure bit decode, bit-exact with the scalar body. The
 # class-bit layout matches the f16/f32 forms above.
 _CMP_CLASS_F64 = (
     '[](auto s, auto m) {'
-    ' using U = util::native<uint64_t>;'
+    ' using U = std::decay_t<decltype(s)>;'
+    ' using M = std::decay_t<decltype(m)>;'
     ' U exp = (s >> 52) & 0x7FFu;'
     ' U mant = s & 0xFFFFFFFFFFFFFull;'
     ' auto sgn = ((s >> 63) & 1u) != 0u;'
@@ -1575,7 +1575,7 @@ def _build_simd_vopc64() -> dict[str, tuple[str, str]]:
     ' util::stdx::where(is_den && !sgn, cls) = 0x080u;'
     ' util::stdx::where(is_norm && !sgn, cls) = 0x100u;'
     ' util::stdx::where(is_inf && !sgn, cls) = 0x200u;'
-    ' auto cls32 = util::stdx::static_simd_cast<util::narrow32<uint32_t>>(cls);'
+    ' auto cls32 = util::stdx::static_simd_cast<M>(cls);'
     ' return (cls32 & m) != 0u; }'
 )
 SIMD_VOPC_CLASS_F64: dict[str, str] = {
@@ -1879,6 +1879,38 @@ def _build_simd_vopc_vop3_f64() -> dict[str, str]:
     'v_xor_b16_vop3': ('uint32_t', '[](auto a, auto b) { return (a ^ b) & 0xFFFFu; }'),
 }
 
+# True16 VOP3 scalar semantics select 16-bit source halves and merge a selected
+# destination half. The generic 32-bit VOP3 SIMD glue overwrites the whole dword.
+SIMD_VOP3_TRUE16_UNSAFE = frozenset(
+    {
+        'v_add_i16_vop3',
+        'v_add_nc_i16_vop3',
+        'v_add_nc_u16_vop3',
+        'v_add_u16_vop3',
+        'v_and_b16_vop3',
+        'v_cmp_eq_i16_vop3',
+        'v_cmp_eq_u16_vop3',
+        'v_cmp_ge_i16_vop3',
+        'v_cmp_ge_u16_vop3',
+        'v_cmp_gt_i16_vop3',
+        'v_cmp_gt_u16_vop3',
+        'v_cmp_le_i16_vop3',
+        'v_cmp_le_u16_vop3',
+        'v_cmp_lt_i16_vop3',
+        'v_cmp_lt_u16_vop3',
+        'v_cmp_ne_i16_vop3',
+        'v_cmp_ne_u16_vop3',
+        'v_mul_lo_u16_vop3',
+        'v_not_b16_vop3',
+        'v_or_b16_vop3',
+        'v_sub_i16_vop3',
+        'v_sub_nc_i16_vop3',
+        'v_sub_nc_u16_vop3',
+        'v_sub_u16_vop3',
+        'v_xor_b16_vop3',
+    }
+)
+
 # VOP3 unary integer ops without a VOP1 twin. Reuse the VOP1 unary glue
 # (operand shape is identical: src0 in, vdst out, 32-bit lanes) — only the
 # probe routing key differs. RDNA3+; CDNA4 does not decode.
@@ -1948,13 +1980,7 @@ def _build_simd_vopc_vop3_f64() -> dict[str, str]:
     'v_rndne_f64_vop3': '[](auto a) { return util::rndne_simd(a); }',
     # sqrt_f64 is correctly-rounded IEEE (scalar uses transcendental::sqrt_f64
     # which is `std::sqrt` after NaN/negative guards); stdx::sqrt matches.
-    'v_sqrt_f64_vop3': (
-        '[](auto a) {'
-        ' auto r = util::stdx::sqrt(a);'
-        ' util::stdx::where(util::stdx::isnan(a), r) = a;'
-        ' util::stdx::where(a < 0.0, r) = std::numeric_limits<double>::quiet_NaN();'
-        ' return r; }'
-    ),
+    'v_sqrt_f64_vop3': ('[](auto a) { return util::sqrt_f64_simd(a); }'),
     # v_fract_f64: scalar = v - std::floor(v); util::floor_simd matches
     # std::floor bit-exact incl. sign-of-zero (NaN-floor(NaN) = NaN; NaN result
     # skipped by the test like any other NaN-result lane).
@@ -2433,26 +2459,8 @@ def _build_simd_vopc_vop3_f64() -> dict[str, str]:
         'uint32_t',
         '[](auto a, auto b, auto c) { return (a & 0xFFFFu) * (b & 0xFFFFu) + c; }',
     ),
-    # v_mad_i16 / v_mad_u16 (and the _legacy twins): 16-bit multiply-add, result
-    # truncated to the low 16 bits and zero-extended. The low 16 bits of the
-    # product+add are independent of operand sign, so masking the operands and
-    # the result to 16 bits reproduces the int16 and uint16 scalar bodies alike.
-    'v_mad_i16_vop3': (
-        'uint32_t',
-        '[](auto a, auto b, auto c) { return ((a & 0xFFFFu) * (b & 0xFFFFu) + (c & 0xFFFFu)) & 0xFFFFu; }',
-    ),
-    'v_mad_u16_vop3': (
-        'uint32_t',
-        '[](auto a, auto b, auto c) { return ((a & 0xFFFFu) * (b & 0xFFFFu) + (c & 0xFFFFu)) & 0xFFFFu; }',
-    ),
-    'v_mad_legacy_i16_vop3': (
-        'uint32_t',
-        '[](auto a, auto b, auto c) { return ((a & 0xFFFFu) * (b & 0xFFFFu) + (c & 0xFFFFu)) & 0xFFFFu; }',
-    ),
-    'v_mad_legacy_u16_vop3': (
-        'uint32_t',
-        '[](auto a, auto b, auto c) { return ((a & 0xFFFFu) * (b & 0xFFFFu) + (c & 0xFFFFu)) & 0xFFFFu; }',
-    ),
+    # v_mad_i16 / v_mad_u16 need true16 VOP3 op_sel handling for each source
+    # and for the destination half, so their executors stay scalar.
     # v_bfe_u32: bitfield extract. off = src1 & 31, w = src2 & 31, result =
     # (src >> off) & ((1<<w)-1). The scalar `if (w==0) return 0` is redundant —
     # ((1u<<w)-1) is already 0 for w==0 — and `w>=32` is dead since w is masked
@@ -2561,6 +2569,8 @@ def _build_simd_vopc_vop3_f64() -> dict[str, str]:
 
 def simd_probe_line(template_name: str) -> str | None:
     """Return the SIMD fast-path probe block for a kernel, or None."""
+    if template_name in SIMD_VOP3_TRUE16_UNSAFE:
+        return None
     if template_name in SIMD_VOP2_CNDMASK:
         return '  ROCJITSU_TRY_SIMD_VOP2_CNDMASK();'
     if template_name in SIMD_VOP3_CNDMASK: