[rocjitsu] Execute packed FP64 basic arithmetic.

emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py

@@ -109,37 +109,39 @@ class _Literal32Widening(Enum):
     F64_HIGH_BITS = 'F64HighBits'
 
 
-# Keep every shipped 64-bit source format explicit. PK_F32 replicates its literal
-# into both DWORDs, while I64 and F64 reinterpret the extension word by type.
-_SIMM32_64BIT_WIDENING = {
-    'FMT_NUM_B32': _Literal32Widening.ZERO_EXTEND,
-    'FMT_NUM_B64': _Literal32Widening.ZERO_EXTEND,
-    'FMT_NUM_BF16': _Literal32Widening.ZERO_EXTEND,
-    'FMT_NUM_BF8': _Literal32Widening.ZERO_EXTEND,
-    'FMT_NUM_F16': _Literal32Widening.ZERO_EXTEND,
-    'FMT_NUM_F32': _Literal32Widening.ZERO_EXTEND,
-    'FMT_NUM_FP8': _Literal32Widening.ZERO_EXTEND,
-    'FMT_NUM_I8': _Literal32Widening.ZERO_EXTEND,
-    'FMT_NUM_M64': _Literal32Widening.ZERO_EXTEND,
-    'FMT_NUM_PK16_U4': _Literal32Widening.ZERO_EXTEND,
-    'FMT_NUM_PK2_B32': _Literal32Widening.ZERO_EXTEND,
-    'FMT_NUM_PK2_F32': _Literal32Widening.REPLICATE_32,
-    'FMT_NUM_PK2_U32': _Literal32Widening.REPLICATE_32,
-    'FMT_NUM_PK4_BF16': _Literal32Widening.ZERO_EXTEND,
-    'FMT_NUM_PK4_F16': _Literal32Widening.ZERO_EXTEND,
-    'FMT_NUM_PK8_BF8': _Literal32Widening.ZERO_EXTEND,
-    'FMT_NUM_PK8_FP8': _Literal32Widening.ZERO_EXTEND,
-    'FMT_NUM_PK8_I8': _Literal32Widening.ZERO_EXTEND,
-    'FMT_NUM_U64': _Literal32Widening.ZERO_EXTEND,
-    'FMT_WMMA_AB_16X16_BF8': _Literal32Widening.ZERO_EXTEND,
-    'FMT_WMMA_AB_16X16_FP8': _Literal32Widening.ZERO_EXTEND,
-    'FMT_WMMA_AB_16X16_IU8': _Literal32Widening.ZERO_EXTEND,
-    'FMT_WMMA_AB_16X32_IU4': _Literal32Widening.ZERO_EXTEND,
-    'FMT_WMMA_AB_16X4_F32': _Literal32Widening.ZERO_EXTEND,
-    'FMT_WMMA_AB_IU4': _Literal32Widening.ZERO_EXTEND,
-    'FMT_WMMA_INDEX_SET2': _Literal32Widening.ZERO_EXTEND,
-    'FMT_NUM_I64': _Literal32Widening.SIGN_EXTEND,
-    'FMT_NUM_F64': _Literal32Widening.F64_HIGH_BITS,
+# Keep every shipped wide-source signature explicit. PK_F32 replicates its
+# literal into both DWORDs, while I64 and F64 reinterpret the extension word by
+# type. Packed F64 is a 128-bit operand containing two broadcast 64-bit values.
+_SIMM32_LITERAL_WIDENING = {
+    (64, 'FMT_NUM_B32'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_NUM_B64'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_NUM_BF16'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_NUM_BF8'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_NUM_F16'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_NUM_F32'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_NUM_FP8'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_NUM_I8'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_NUM_M64'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_NUM_PK16_U4'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_NUM_PK2_B32'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_NUM_PK2_F32'): _Literal32Widening.REPLICATE_32,
+    (64, 'FMT_NUM_PK2_U32'): _Literal32Widening.REPLICATE_32,
+    (64, 'FMT_NUM_PK4_BF16'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_NUM_PK4_F16'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_NUM_PK8_BF8'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_NUM_PK8_FP8'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_NUM_PK8_I8'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_NUM_U64'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_WMMA_AB_16X16_BF8'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_WMMA_AB_16X16_FP8'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_WMMA_AB_16X16_IU8'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_WMMA_AB_16X32_IU4'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_WMMA_AB_16X4_F32'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_WMMA_AB_IU4'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_WMMA_INDEX_SET2'): _Literal32Widening.ZERO_EXTEND,
+    (64, 'FMT_NUM_I64'): _Literal32Widening.SIGN_EXTEND,
+    (64, 'FMT_NUM_F64'): _Literal32Widening.F64_HIGH_BITS,
+    (128, 'FMT_NUM_PK2_F64'): _Literal32Widening.F64_HIGH_BITS,
 }
 
 _LITERAL_ENCODING_OPERANDS = {
@@ -1209,7 +1211,7 @@ def _literal_operand_from_expr_stmt(
             )
         if operand_type == 'OPR_SIMM16' or opnd.size == 16:
             literal_expr = f'({literal_expr} & 0xFFFFu)'
-        widening = CodeGenerator._literal_operand_64bit_widening(
+        widening = CodeGenerator._literal_operand_simm32_widening(
             opnd, operand_type, arch_name, inst_name, enc_name
         )
         if widening is not None:
@@ -1224,23 +1226,29 @@ def _literal_operand_from_expr_stmt(
         )
 
     @staticmethod
-    def _literal_operand_64bit_widening(
+    def _literal_operand_simm32_widening(
         opnd: Operand,
         operand_type: str,
         arch_name: str,
         inst_name: str,
         enc_name: str,
     ) -> _Literal32Widening | None:
-        if operand_type != 'OPR_SIMM32' or not opnd.is_input or opnd.size != 64:
+        if operand_type != 'OPR_SIMM32' or not opnd.is_input:
             return None
+        signature = (opnd.size, opnd.data_format_name)
         try:
-            return _SIMM32_64BIT_WIDENING[opnd.data_format_name]
+            return _SIMM32_LITERAL_WIDENING[signature]
         except KeyError as exc:
+            # Every 64-bit SIMM32 input requires an explicit policy. Wider
+            # packed inputs retain the default immediate semantics unless
+            # listed above.
+            if opnd.size != 64:
+                return None
             data_format = opnd.data_format_name or '<missing>'
             raise ValueError(
                 f'architecture {arch_name!r}, instruction {inst_name!r}, encoding '
-                f'{enc_name!r}: 64-bit SIMM32 input operand {opnd.name!r} has '
-                f'unsupported data format {data_format!r}'
+                f'{enc_name!r}: SIMM32 input operand {opnd.name!r} has no widening '
+                f'policy for size {opnd.size} and data format {data_format!r}'
             ) from exc
 
     @staticmethod
@@ -10383,12 +10391,12 @@ def gen_insts(self) -> None:
                             f'<< "{inst.name} requires its GLOBAL segment";'
                         )
 
-                    # LLVM's public gfx1251 profiles define the packed U64
-                    # operations without op_sel.  The profile and corresponding
-                    # MC vectors are permanently linked here:
-                    # https://github.com/llvm/llvm-project/blob/3bcd9a803184e2d3657b9d5cc2a1773e9ce0f116/llvm/lib/Target/AMDGPU/VOP3PInstructions.td#L147-L162
-                    # https://github.com/llvm/llvm-project/blob/3bcd9a803184e2d3657b9d5cc2a1773e9ce0f116/llvm/test/MC/AMDGPU/gfx1251_asm_vop3p.s#L257-L403
-                    # https://github.com/llvm/llvm-project/blob/551d5172dd3902efbce5f4720b75bfc4e6441dc8/llvm/lib/Target/AMDGPU/SIRegisterInfo.td#L887-L917
+                    # LLVM's public gfx1251 profiles define the packed U64 and
+                    # F64 binary operations without op_sel.  The profile and
+                    # corresponding MC vectors are permanently linked here:
+                    # https://github.com/llvm/llvm-project/blob/3bcd9a803184e2d3657b9d5cc2a1773e9ce0f116/llvm/lib/Target/AMDGPU/VOP3PInstructions.td#L130-L162
+                    # https://github.com/llvm/llvm-project/blob/3bcd9a803184e2d3657b9d5cc2a1773e9ce0f116/llvm/test/MC/AMDGPU/gfx1251_asm_vop3p.s#L1-L403
+                    # https://github.com/llvm/llvm-project/blob/551d5172dd3902efbce5f4720b75bfc4e6441dc8/llvm/lib/Target/AMDGPU/SIRegisterInfo.td#L875-L920
                     # https://github.com/llvm/llvm-project/blob/551d5172dd3902efbce5f4720b75bfc4e6441dc8/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp#L2133-L2143
                     # These references establish the operand profile and
                     # encoding, not execution ordering for combined modifiers.
@@ -10400,28 +10408,34 @@ def gen_insts(self) -> None:
                     # tuple's first selector and its width-independent source
                     # selector set.
                     if inst_sem is not None and inst_sem.semantic_class in {
+                        'pk_binop_f64',
                         'pk_binop_u64',
                         'pk_lshl_add_u64',
                     }:
                         raw_inst = (
                             f'reinterpret_cast<const {factory_op_encoding}*>(inst)'
                         )
+                        layout_name = (
+                            'packed F64'
+                            if inst_sem.semantic_class == 'pk_binop_f64'
+                            else 'packed U64'
+                        )
                         factory_validation_parts.append(
                             f'if ({raw_inst}->opsel != 0u || '
                             f'{raw_inst}->opsel_hi != 3u || '
                             f'{raw_inst}->opsel_hi_2 != 1u) '
                             f'[[unlikely]] return emit_error.emit() << "{inst.name} has an invalid '
-                            'packed U64 element layout";'
+                            f'{layout_name} element layout";'
                         )
-                        if inst_sem.semantic_class == 'pk_binop_u64':
+                        if inst_sem.semantic_class in {'pk_binop_f64', 'pk_binop_u64'}:
                             factory_validation_parts.append(
                                 f'if ({raw_inst}->src2 != 128u || '
                                 f'({raw_inst}->neg & 4u) != 0u || '
                                 f'({raw_inst}->neg_hi & 4u) != 0u) '
                                 f'[[unlikely]] return emit_error.emit() << "{inst.name} has an invalid '
                                 'unused src2 encoding";'
                             )
-                        else:
+                        if inst_sem.semantic_class == 'pk_lshl_add_u64':
                             factory_validation_parts.append(
                                 f'if ({raw_inst}->clamp != 0u || '
                                 f'{raw_inst}->neg != 0u || '
@@ -10449,7 +10463,7 @@ def gen_insts(self) -> None:
                                 invalid_sgpr_span = f'{raw_value} > {max_sgpr}u'
                                 if opnd.size == 128:
                                     # SGPR104_128 is a named VS_128 member:
-                                    # s104:s105 provide the scalar U64 value,
+                                    # s104:s105 provide the scalar 64-bit value,
                                     # while vcc_lo:vcc_hi complete the tuple.
                                     invalid_sgpr_span = (
                                         f'({invalid_sgpr_span} && {raw_value} != 104u)'
@@ -10509,7 +10523,7 @@ def gen_insts(self) -> None:
                                 factory_validation_parts.append(
                                     f'if (!({valid_width_specific})) '
                                     f'[[unlikely]] return emit_error.emit() << "{inst.name} has an invalid '
-                                    f'{opnd.name} packed U64 source selector";'
+                                    f'{opnd.name} {layout_name} source selector";'
                                 )
 
                     # Flat segment-aware operands: adjust addr width and add

emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py

@@ -120,6 +120,7 @@ def _register_handlers() -> None:
         gen_pk_binop_f32,
         gen_pk_ternary_f32,
         gen_pk_binop_u64,
+        gen_pk_binop_f64,
         gen_pk_lshl_add_u64,
         gen_pk_mov_b32,
         gen_mad_mix_f32,
@@ -341,6 +342,7 @@ def _register_handlers() -> None:
         use_cdna5_helpers=c.arch_name == 'cdna5',
     )
     DISPATCH['pk_binop_u64'] = lambda c: gen_pk_binop_u64(c.dst_ops, c.src_ops, c.op)
+    DISPATCH['pk_binop_f64'] = lambda c: gen_pk_binop_f64(c.dst_ops, c.src_ops, c.op)
     DISPATCH['pk_lshl_add_u64'] = lambda c: gen_pk_lshl_add_u64(c.dst_ops, c.src_ops)
     DISPATCH['pk_mov_b32'] = lambda c: gen_pk_mov_b32(
         c.dst_ops,

emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py

@@ -734,6 +734,50 @@ def gen_pk_binop_u64(dst: list[str], src: list[str], op: str) -> str:
     return '\n'.join(L)
 
 
+def gen_pk_binop_f64(dst: list[str], src: list[str], op: str) -> str:
+    """Generate two-element packed F64 add/mul/minNum/maxNum execution."""
+    operations = {
+        'add': 'Add',
+        'mul': 'Multiply',
+        'max_num': 'MaximumNumber',
+        'min_num': 'MinimumNumber',
+    }
+    try:
+        operation = operations[op]
+    except KeyError as exc:
+        raise ValueError(f'unsupported packed F64 binary operation: {op}') from exc
+
+    d, s0, s1 = dst[0], src[0], src[1]
+    return '\n'.join(
+        [
+            '  uint64_t exec = wf.exec();',
+            '  for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) {',
+            '    if (!(exec & (1ULL << lane))) continue;',
+            f'    const auto lhs = read_pk_u64_pair({s0}, wf, lane);',
+            f'    const auto rhs = read_pk_u64_pair({s1}, wf, lane);',
+            '    constexpr uint64_t kSignBit = 0x8000000000000000ULL;',
+            '    const auto apply = [&](uint64_t lhs_bits, uint64_t rhs_bits,',
+            '                           bool negate_lhs, bool negate_rhs) {',
+            '      if (negate_lhs) lhs_bits ^= kSignBit;',
+            '      if (negate_rhs) rhs_bits ^= kSignBit;',
+            '      uint64_t result = amdgpu::fp_mode::binary_f64(',
+            f'          lhs_bits, rhs_bits, amdgpu::fp_mode::BinaryF64Op::{operation},',
+            '          wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64());',
+            '      return amdgpu::fp_mode::finish_f64(',
+            '          result, wf.fp_round_mode_f16_f64(), 0, inst_.clamp,',
+            '          amdgpu::floating_clamp_nan_to_zero(wf));',
+            '    };',
+            '    const uint64_t result_lo =',
+            '        apply(lhs.lo, rhs.lo, (inst_.neg & 1u) != 0, (inst_.neg & 2u) != 0);',
+            '    const uint64_t result_hi = apply(lhs.hi, rhs.hi,',
+            '                                     (inst_.neg_hi & 1u) != 0,',
+            '                                     (inst_.neg_hi & 2u) != 0);',
+            f'    write_pk_u64_pair({d}, wf, lane, {{result_lo, result_hi}});',
+            '  }',
+        ]
+    )
+
+
 def gen_pk_lshl_add_u64(dst: list[str], src: list[str]) -> str:
     """Generate packed U64 shift-left-add over two independent elements.
 

emulation/rocjitsu/lib/python/amdisa/semantics.py

@@ -1603,6 +1603,19 @@ def _derive_vop3p(name: str) -> InstructionSemantics | None:
         return InstructionSemantics(
             name, 'pk_binop_f32', operation='add', data_type='f32'
         )
+    packed_f64_binary = {
+        'V_PK_ADD_F64': 'add',
+        'V_PK_MUL_F64': 'mul',
+        'V_PK_MAX_NUM_F64': 'max_num',
+        'V_PK_MIN_NUM_F64': 'min_num',
+    }
+    if name in packed_f64_binary:
+        return InstructionSemantics(
+            name,
+            'pk_binop_f64',
+            operation=packed_f64_binary[name],
+            data_type='f64',
+        )
     if name in ('V_PK_ADD_NC_U64', 'V_PK_SUB_NC_U64'):
         operation = 'add' if name == 'V_PK_ADD_NC_U64' else 'sub'
         return InstructionSemantics(

emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py

@@ -5262,6 +5262,78 @@ def test_gfx1251_packed_u64_decode_rejects_undefined_layouts_and_register_tuples
     assert 'make_after_selector_validation' not in lshl_constructor
 
 
+def test_gfx1251_packed_f64_decode_rejects_undefined_layouts_and_register_tuples(
+    gfx1250_generated_root: Path, tmp_path: Path
+):
+    args = SimpleNamespace(
+        isafiles=[f'cdna5:{_mrisa_dir() / "amdgpu_isa_cdna5.xml"}'],
+        isa_additions=[f'cdna5:{_mrisa_dir() / "amdgpu_isa_cdna5_gfx1251_delta.xml"}'],
+        isa_variants=[f'cdna5:{_mrisa_dir() / "amdgpu_isa_cdna5_variants.json"}'],
+        gen_isas=True,
+        gen_dbt=False,
+        isa_output=str(tmp_path),
+        dbt_output=None,
+    )
+    _run(args)
+
+    for generated_root in (gfx1250_generated_root, tmp_path / 'cdna5'):
+        source = (generated_root / 'vop3p.cpp').read_text()
+        for class_name in (
+            'VPkAddF64Vop3p',
+            'VPkMulF64Vop3p',
+            'VPkMaxNumF64Vop3p',
+            'VPkMinNumF64Vop3p',
+        ):
+            body = _generated_decode_body(source, class_name)
+            assert 'has an invalid packed F64 element layout' in body
+            assert 'has an invalid unused src2 encoding' in body
+            assert '->neg & 4u) != 0u' in body
+            assert '->neg_hi & 4u) != 0u' in body
+            assert 'vdst register tuple that exceeds the selector range' in body
+            assert 'src0 register tuple that exceeds the selector range' in body
+            assert 'src1 register tuple that exceeds the selector range' in body
+            assert 'invalid vdst register tuple alignment' in body
+            assert 'invalid src0 register tuple alignment' in body
+            assert 'invalid src1 register tuple alignment' in body
+            assert 'invalid src0 packed F64 source selector' in body
+            assert 'invalid src1 packed F64 source selector' in body
+            for operand_name in ('src0', 'src1'):
+                assert f'{operand_name} != 104u' in body
+                assert f'{operand_name} == 104u' in body
+                assert f'{operand_name} <= 100u' in body
+                assert f'{operand_name} >= 108u' in body
+                assert f'{operand_name} <= 120u' in body
+                assert f'{operand_name} == 124u' in body
+                assert f'{operand_name} >= 128u' in body
+                assert f'{operand_name} <= 208u' in body
+                assert f'{operand_name} >= 240u' in body
+                assert f'{operand_name} <= 248u' in body
+                assert f'{operand_name} == 255u' in body
+                assert f'{operand_name} >= 256u' in body
+                assert f'{operand_name} <= 508u' in body
+                for invalid_selector in (230, 231, 235, 236, 253):
+                    assert f'{operand_name} == {invalid_selector}u' not in body
+            assert 'does not support source modifiers or clamp' not in body
+
+
+def test_gfx1251_packed_f64_literals_use_f64_high_bits_widening(
+    gfx1250_generated_root: Path,
+):
+    source = (gfx1250_generated_root / 'vop3p.cpp').read_text()
+
+    for class_name in (
+        'VPkFmaF64Vop3p',
+        'VPkMulF64Vop3p',
+        'VPkAddF64Vop3p',
+        'VPkMaxNumF64Vop3p',
+        'VPkMinNumF64Vop3p',
+    ):
+        constructor = source.split(f'{class_name}::{class_name}(', 1)[1].split(
+            '\n}', 1
+        )[0]
+        assert 'Operand::Literal32Widening::F64HighBits' in constructor
+
+
 def test_gfx1250_matrix_codegen_uses_public_opsel_hi_2_field(
     gfx1250_generated_root: Path,
 ):
@@ -5348,10 +5420,6 @@ def test_cdna5_variant_execution_callback_inventory(
 ) -> None:
     model_only_classes = (
         'VPkFmaF64Vop3p',
-        'VPkMulF64Vop3p',
-        'VPkAddF64Vop3p',
-        'VPkMaxNumF64Vop3p',
-        'VPkMinNumF64Vop3p',
         'VWmmaF6416x16x4F64Vop3p',
     )
     header = (gfx1250_generated_root / 'vop3p.h').read_text()
@@ -5372,8 +5440,12 @@ def test_cdna5_variant_execution_callback_inventory(
         assert class_name not in execution_source
 
     executable_classes = (
+        'VPkMulF64Vop3p',
+        'VPkAddF64Vop3p',
         'VPkAddNcU64Vop3p',
         'VPkSubNcU64Vop3p',
+        'VPkMaxNumF64Vop3p',
+        'VPkMinNumF64Vop3p',
         'VPkLshlAddU64Vop3p',
     )
     for executable_class in executable_classes: