From e4fc83bc07d817ef25f3dc51d6f98017c38c127c Mon Sep 17 00:00:00 2001 From: hanhanW Date: Mon, 24 Aug 2026 02:07:16 -0700 Subject: [PATCH 1/8] [rocjitsu] Execute packed FP64 basic arithmetic. Enable instruction-level execution for gfx1251 V_PK_ADD_F64, V_PK_MUL_F64, V_PK_MAX_NUM_F64, and V_PK_MIN_NUM_F64. These instructions were previously model-only, so valid decoded instructions could not produce architectural register results. Generate callbacks that route both packed elements through the shared MODE-aware F64 helpers. This honors rounding and denormal modes, EXEC masking, independent low/high source negation, clamp, packed-FP64 scalar propagation, literals, inline constants, and overlapping register tuples. Validate the fixed packed layout and public LLVM width-specific source and register classes before installing each callback. Tests exercise the public LLVM encodings and source forms, boundary selectors and tuples, FP modes, NaN and signed-zero behavior, modifiers, inactive lanes, aliasing, and decoder rejection paths. Keep V_PK_FMA_F64 and V_WMMA_F64_16X16X4_F64 model-only. gfx1251 still does not advertise full simulator execution, so this change provides decode-to-register execution rather than simulator or AQL end-to-end support. --- .../lib/python/amdisa/codegen/_generator.py | 32 +- .../python/amdisa/codegen/execute/__init__.py | 2 + .../python/amdisa/codegen/execute/packed.py | 44 + .../rocjitsu/lib/python/amdisa/semantics.py | 13 + .../tests/test_generator_profile_gates.py | 65 +- .../amdisa/tests/test_packed_codegen.py | 31 + .../python/amdisa/tests/test_sema_derive.py | 14 + .../generated/cdna5/execution_backend.h | 4 + .../cdna5/execution_backend_exec.cpp | 4 + .../isa/arch/amdgpu/generated/cdna5/vop3p.cpp | 365 +++++++- .../isa/arch/amdgpu/generated/cdna5/vop3p.h | 4 + .../amdgpu/generated/cdna5/vop3p_exec.cpp | 104 +++ .../rocjitsu/isa/arch/amdgpu/shared/fp_mode.h | 42 + .../rocjitsu/tests/cdna5_execution_test.cpp | 831 ++++++++++++++++++ .../isa/amdgpu_isa_cdna5_variants.json | 4 - 15 files changed, 1503 insertions(+), 56 deletions(-) diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py index 94982d63969c..5452c96beb2f 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py @@ -124,6 +124,7 @@ class _Literal32Widening(Enum): 'FMT_NUM_PK16_U4': _Literal32Widening.ZERO_EXTEND, 'FMT_NUM_PK2_B32': _Literal32Widening.ZERO_EXTEND, 'FMT_NUM_PK2_F32': _Literal32Widening.REPLICATE_32, + 'FMT_NUM_PK2_F64': _Literal32Widening.F64_HIGH_BITS, 'FMT_NUM_PK2_U32': _Literal32Widening.REPLICATE_32, 'FMT_NUM_PK4_BF16': _Literal32Widening.ZERO_EXTEND, 'FMT_NUM_PK4_F16': _Literal32Widening.ZERO_EXTEND, @@ -1231,7 +1232,10 @@ def _literal_operand_64bit_widening( inst_name: str, enc_name: str, ) -> _Literal32Widening | None: - if operand_type != 'OPR_SIMM32' or not opnd.is_input or opnd.size != 64: + is_64bit_value = opnd.size == 64 or ( + opnd.size == 128 and opnd.data_format_name == 'FMT_NUM_PK2_F64' + ) + if operand_type != 'OPR_SIMM32' or not opnd.is_input or not is_64bit_value: return None try: return _SIMM32_64BIT_WIDENING[opnd.data_format_name] @@ -10383,12 +10387,12 @@ def gen_insts(self) -> None: f'<< "{inst.name} requires its GLOBAL segment";' ) - # LLVM's public gfx1251 profiles define the packed U64 - # operations without op_sel. The profile and corresponding - # MC vectors are permanently linked here: - # https://github.com/llvm/llvm-project/blob/3bcd9a803184e2d3657b9d5cc2a1773e9ce0f116/llvm/lib/Target/AMDGPU/VOP3PInstructions.td#L147-L162 - # https://github.com/llvm/llvm-project/blob/3bcd9a803184e2d3657b9d5cc2a1773e9ce0f116/llvm/test/MC/AMDGPU/gfx1251_asm_vop3p.s#L257-L403 - # https://github.com/llvm/llvm-project/blob/551d5172dd3902efbce5f4720b75bfc4e6441dc8/llvm/lib/Target/AMDGPU/SIRegisterInfo.td#L887-L917 + # LLVM's public gfx1251 profiles define the packed U64 and + # F64 binary operations without op_sel. The profile and + # corresponding MC vectors are permanently linked here: + # https://github.com/llvm/llvm-project/blob/3bcd9a803184e2d3657b9d5cc2a1773e9ce0f116/llvm/lib/Target/AMDGPU/VOP3PInstructions.td#L130-L162 + # https://github.com/llvm/llvm-project/blob/3bcd9a803184e2d3657b9d5cc2a1773e9ce0f116/llvm/test/MC/AMDGPU/gfx1251_asm_vop3p.s#L1-L403 + # https://github.com/llvm/llvm-project/blob/551d5172dd3902efbce5f4720b75bfc4e6441dc8/llvm/lib/Target/AMDGPU/SIRegisterInfo.td#L875-L920 # https://github.com/llvm/llvm-project/blob/551d5172dd3902efbce5f4720b75bfc4e6441dc8/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp#L2133-L2143 # These references establish the operand profile and # encoding, not execution ordering for combined modifiers. @@ -10400,20 +10404,26 @@ def gen_insts(self) -> None: # tuple's first selector and its width-independent source # selector set. if inst_sem is not None and inst_sem.semantic_class in { + 'pk_binop_f64', 'pk_binop_u64', 'pk_lshl_add_u64', }: raw_inst = ( f'reinterpret_cast(inst)' ) + layout_name = ( + 'packed F64' + if inst_sem.semantic_class == 'pk_binop_f64' + else 'packed U64' + ) factory_validation_parts.append( f'if ({raw_inst}->opsel != 0u || ' f'{raw_inst}->opsel_hi != 3u || ' f'{raw_inst}->opsel_hi_2 != 1u) ' f'[[unlikely]] return emit_error.emit() << "{inst.name} has an invalid ' - 'packed U64 element layout";' + f'{layout_name} element layout";' ) - if inst_sem.semantic_class == 'pk_binop_u64': + if inst_sem.semantic_class in {'pk_binop_f64', 'pk_binop_u64'}: factory_validation_parts.append( f'if ({raw_inst}->src2 != 128u || ' f'({raw_inst}->neg & 4u) != 0u || ' @@ -10421,7 +10431,7 @@ def gen_insts(self) -> None: f'[[unlikely]] return emit_error.emit() << "{inst.name} has an invalid ' 'unused src2 encoding";' ) - else: + if inst_sem.semantic_class == 'pk_lshl_add_u64': factory_validation_parts.append( f'if ({raw_inst}->clamp != 0u || ' f'{raw_inst}->neg != 0u || ' @@ -10509,7 +10519,7 @@ def gen_insts(self) -> None: factory_validation_parts.append( f'if (!({valid_width_specific})) ' f'[[unlikely]] return emit_error.emit() << "{inst.name} has an invalid ' - f'{opnd.name} packed U64 source selector";' + f'{opnd.name} {layout_name} source selector";' ) # Flat segment-aware operands: adjust addr width and add diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py index 74f8ab705a1b..1c8407d47519 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py @@ -118,6 +118,7 @@ def _register_handlers() -> None: gen_pk_fmac_vop2, gen_pk_fmac_vop3, gen_pk_binop_f32, + gen_pk_binop_f64, gen_pk_ternary_f32, gen_pk_binop_u64, gen_pk_lshl_add_u64, @@ -341,6 +342,7 @@ def _register_handlers() -> None: use_cdna5_helpers=c.arch_name == 'cdna5', ) DISPATCH['pk_binop_u64'] = lambda c: gen_pk_binop_u64(c.dst_ops, c.src_ops, c.op) + DISPATCH['pk_binop_f64'] = lambda c: gen_pk_binop_f64(c.dst_ops, c.src_ops, c.op) DISPATCH['pk_lshl_add_u64'] = lambda c: gen_pk_lshl_add_u64(c.dst_ops, c.src_ops) DISPATCH['pk_mov_b32'] = lambda c: gen_pk_mov_b32( c.dst_ops, diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py index f22ac4088b5a..82a488b6cf4a 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py @@ -734,6 +734,50 @@ def gen_pk_binop_u64(dst: list[str], src: list[str], op: str) -> str: return '\n'.join(L) +def gen_pk_binop_f64(dst: list[str], src: list[str], op: str) -> str: + """Generate two-element packed F64 add/mul/minNum/maxNum execution.""" + operations = { + 'add': 'Add', + 'mul': 'Multiply', + 'max_num': 'MaximumNumber', + 'min_num': 'MinimumNumber', + } + try: + operation = operations[op] + except KeyError as exc: + raise ValueError(f'unsupported packed F64 binary operation: {op}') from exc + + d, s0, s1 = dst[0], src[0], src[1] + return '\n'.join( + [ + ' uint64_t exec = wf.exec();', + ' for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) {', + ' if (!(exec & (1ULL << lane))) continue;', + f' const auto lhs = read_pk_u64_pair({s0}, wf, lane);', + f' const auto rhs = read_pk_u64_pair({s1}, wf, lane);', + ' constexpr uint64_t kSignBit = 0x8000000000000000ULL;', + ' const auto apply = [&](uint64_t lhs_bits, uint64_t rhs_bits,', + ' bool negate_lhs, bool negate_rhs) {', + ' if (negate_lhs) lhs_bits ^= kSignBit;', + ' if (negate_rhs) rhs_bits ^= kSignBit;', + ' uint64_t result = amdgpu::fp_mode::binary_f64(', + f' lhs_bits, rhs_bits, amdgpu::fp_mode::BinaryF64Op::{operation},', + ' wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64());', + ' return amdgpu::fp_mode::finish_f64(', + ' result, wf.fp_round_mode_f16_f64(), 0, inst_.clamp,', + ' amdgpu::floating_clamp_nan_to_zero(wf));', + ' };', + ' const uint64_t result_lo =', + ' apply(lhs.lo, rhs.lo, (inst_.neg & 1u) != 0, (inst_.neg & 2u) != 0);', + ' const uint64_t result_hi = apply(lhs.hi, rhs.hi,', + ' (inst_.neg_hi & 1u) != 0,', + ' (inst_.neg_hi & 2u) != 0);', + f' write_pk_u64_pair({d}, wf, lane, {{result_lo, result_hi}});', + ' }', + ] + ) + + def gen_pk_lshl_add_u64(dst: list[str], src: list[str]) -> str: """Generate packed U64 shift-left-add over two independent elements. diff --git a/emulation/rocjitsu/lib/python/amdisa/semantics.py b/emulation/rocjitsu/lib/python/amdisa/semantics.py index fa7752f07422..24e8a1382be1 100644 --- a/emulation/rocjitsu/lib/python/amdisa/semantics.py +++ b/emulation/rocjitsu/lib/python/amdisa/semantics.py @@ -1603,6 +1603,19 @@ def _derive_vop3p(name: str) -> InstructionSemantics | None: return InstructionSemantics( name, 'pk_binop_f32', operation='add', data_type='f32' ) + packed_f64_binary = { + 'V_PK_ADD_F64': 'add', + 'V_PK_MUL_F64': 'mul', + 'V_PK_MAX_NUM_F64': 'max_num', + 'V_PK_MIN_NUM_F64': 'min_num', + } + if name in packed_f64_binary: + return InstructionSemantics( + name, + 'pk_binop_f64', + operation=packed_f64_binary[name], + data_type='f64', + ) if name in ('V_PK_ADD_NC_U64', 'V_PK_SUB_NC_U64'): operation = 'add' if name == 'V_PK_ADD_NC_U64' else 'sub' return InstructionSemantics( diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py index 745e45fda008..ef4424d0a2ba 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py @@ -5262,6 +5262,63 @@ def test_gfx1251_packed_u64_decode_rejects_undefined_layouts_and_register_tuples assert 'make_after_selector_validation' not in lshl_constructor +def test_gfx1251_packed_f64_decode_rejects_undefined_layouts_and_register_tuples( + gfx1250_generated_root: Path, +): + source = (gfx1250_generated_root / 'vop3p.cpp').read_text() + + for class_name in ( + 'VPkAddF64Vop3p', + 'VPkMulF64Vop3p', + 'VPkMaxNumF64Vop3p', + 'VPkMinNumF64Vop3p', + ): + body = _generated_decode_body(source, class_name) + assert 'has an invalid packed F64 element layout' in body + assert 'has an invalid unused src2 encoding' in body + assert 'vdst register tuple that exceeds the selector range' in body + assert 'src0 register tuple that exceeds the selector range' in body + assert 'src1 register tuple that exceeds the selector range' in body + assert 'invalid vdst register tuple alignment' in body + assert 'invalid src0 register tuple alignment' in body + assert 'invalid src1 register tuple alignment' in body + assert 'invalid src0 packed F64 source selector' in body + assert 'invalid src1 packed F64 source selector' in body + for operand_name in ('src0', 'src1'): + assert f'{operand_name} <= 100u' in body + assert f'{operand_name} >= 108u' in body + assert f'{operand_name} <= 120u' in body + assert f'{operand_name} == 124u' in body + assert f'{operand_name} >= 128u' in body + assert f'{operand_name} <= 208u' in body + assert f'{operand_name} >= 240u' in body + assert f'{operand_name} <= 248u' in body + assert f'{operand_name} == 255u' in body + assert f'{operand_name} >= 256u' in body + assert f'{operand_name} <= 508u' in body + for invalid_selector in (230, 231, 235, 236, 253): + assert f'{operand_name} == {invalid_selector}u' not in body + assert 'does not support source modifiers or clamp' not in body + + +def test_gfx1251_packed_f64_literals_use_f64_high_bits_widening( + gfx1250_generated_root: Path, +): + source = (gfx1250_generated_root / 'vop3p.cpp').read_text() + + for class_name in ( + 'VPkFmaF64Vop3p', + 'VPkMulF64Vop3p', + 'VPkAddF64Vop3p', + 'VPkMaxNumF64Vop3p', + 'VPkMinNumF64Vop3p', + ): + constructor = source.split(f'{class_name}::{class_name}(', 1)[1].split( + '\n}', 1 + )[0] + assert 'Operand::Literal32Widening::F64HighBits' in constructor + + def test_gfx1250_matrix_codegen_uses_public_opsel_hi_2_field( gfx1250_generated_root: Path, ): @@ -5348,10 +5405,6 @@ def test_cdna5_variant_execution_callback_inventory( ) -> None: model_only_classes = ( 'VPkFmaF64Vop3p', - 'VPkMulF64Vop3p', - 'VPkAddF64Vop3p', - 'VPkMaxNumF64Vop3p', - 'VPkMinNumF64Vop3p', 'VWmmaF6416x16x4F64Vop3p', ) header = (gfx1250_generated_root / 'vop3p.h').read_text() @@ -5372,8 +5425,12 @@ def test_cdna5_variant_execution_callback_inventory( assert class_name not in execution_source executable_classes = ( + 'VPkMulF64Vop3p', + 'VPkAddF64Vop3p', 'VPkAddNcU64Vop3p', 'VPkSubNcU64Vop3p', + 'VPkMaxNumF64Vop3p', + 'VPkMinNumF64Vop3p', 'VPkLshlAddU64Vop3p', ) for executable_class in executable_classes: diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py index a0e766cf843e..92325fbf5b34 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py @@ -13,6 +13,7 @@ gen_mad_mix_lo_hi, gen_pk_binop, gen_pk_binop_f32, + gen_pk_binop_f64, gen_pk_fmac_vop2, gen_pk_fmac_vop3, gen_pk_binop_u64, @@ -394,6 +395,36 @@ def test_pk_u64_binop_rejects_unknown_operation(): gen_pk_binop_u64(['vdst'], ['src0', 'src1'], 'mul') +def test_pk_f64_binops_use_mode_aware_two_element_execution(): + expected_operations = { + 'add': 'Add', + 'mul': 'Multiply', + 'max_num': 'MaximumNumber', + 'min_num': 'MinimumNumber', + } + for operation, enum_name in expected_operations.items(): + cpp = gen_pk_binop_f64(['vdst'], ['src0', 'src1'], operation) + + assert 'const auto lhs = read_pk_u64_pair(src0, wf, lane);' in cpp + assert 'const auto rhs = read_pk_u64_pair(src1, wf, lane);' in cpp + assert f'BinaryF64Op::{enum_name}' in cpp + assert cpp.count('amdgpu::fp_mode::binary_f64(') == 1 + assert 'wf.fp_round_mode_f16_f64()' in cpp + assert 'wf.fp_denorm_mode_f16_f64()' in cpp + assert 'lhs_bits ^= kSignBit' in cpp + assert 'rhs_bits ^= kSignBit' in cpp + assert 'amdgpu::fp_mode::finish_f64(' in cpp + assert 'inst_.clamp' in cpp + assert 'write_pk_u64_pair(vdst, wf, lane, {result_lo, result_hi});' in cpp + + +def test_pk_f64_binop_rejects_unknown_operation(): + import pytest + + with pytest.raises(ValueError, match='unsupported packed F64 binary operation'): + gen_pk_binop_f64(['vdst'], ['src0', 'src1'], 'divide') + + def test_renamed_vop3p_packed_f32_probe_passes_profile_selectors(): probe = vop3p_local_simd_probe_line('v_pk_add_f32_vop3p', ('opsel', 'opsel_hi')) diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py index a1fe479377e1..1e78a4149a0f 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py @@ -2661,6 +2661,20 @@ def test_pk_u64_add_sub_have_distinct_binary_semantics(self): assert sem.operation == operation assert sem.data_type == 'u64' + def test_pk_f64_basic_arithmetic_has_distinct_binary_semantics(self): + for name, operation in ( + ('V_PK_ADD_F64', 'add'), + ('V_PK_MUL_F64', 'mul'), + ('V_PK_MAX_NUM_F64', 'max_num'), + ('V_PK_MIN_NUM_F64', 'min_num'), + ): + sem = derive_semantics(name, 'ENC_VOP3P') + + assert sem is not None + assert sem.semantic_class == 'pk_binop_f64' + assert sem.operation == operation + assert sem.data_type == 'f64' + def test_pk_mov_b32(self): sem = _FakeSem('V_PK_MOV_B32', 'pk_mov_b32') block = derive_sema_block(sem) diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/execution_backend.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/execution_backend.h index 93e8e8577ec7..5cb50cdad650 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/execution_backend.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/execution_backend.h @@ -1269,8 +1269,12 @@ enum class InstructionExecutionId : size_t { VWmmaF1616x16x128Bf8Fp8Vop3p, VWmmaF1616x16x128Bf8Bf8Vop3p, VWmmaF3232x16x128F4Vop3p, + VPkMulF64Vop3p, + VPkAddF64Vop3p, VPkAddNcU64Vop3p, VPkSubNcU64Vop3p, + VPkMaxNumF64Vop3p, + VPkMinNumF64Vop3p, VPkLshlAddU64Vop3p, VWmmaScaleF32Vop3px2, DsAddU32Vds, diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/execution_backend_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/execution_backend_exec.cpp index 470034d263de..6666a0be8994 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/execution_backend_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/execution_backend_exec.cpp @@ -1276,8 +1276,12 @@ constexpr InstructionCallbackTable kInstructionCallbacks{{ &execute_with_backend, &execute_with_backend, &execute_with_backend, + &execute_with_backend, + &execute_with_backend, &execute_with_backend, &execute_with_backend, + &execute_with_backend, + &execute_with_backend, &execute_with_backend, &execute_with_backend, &execute_with_backend, diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.cpp index ad19e83e9dbd..3be41e86ae7c 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.cpp @@ -4183,17 +4183,20 @@ VPkFmaF64Vop3p::VPkFmaF64Vop3p(const MachineInst *inst) num_src_ = 3; num_dst_ = 1; if (reinterpret_cast(inst)->src0 == 255) - src0 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src0 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); if (reinterpret_cast(inst)->src1 == 255) - src1 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src1 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); if (reinterpret_cast(inst)->src2 == 255) - src2 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src2 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); vdst.set_vgpr_msb_role(amdgpu::VgprMsbRole::Dst); src0.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src0); src1.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src1); @@ -4217,7 +4220,8 @@ DecodeResult decodeVPkFmaF64Vop3p(const MachineInst *opcode, const DecodeErrorEm } // namespace detail VPkMulF64Vop3p::VPkMulF64Vop3p(const MachineInst *inst) - : Vop3p("v_pk_mul_f64", reinterpret_cast(inst), nullptr), + : Vop3p("v_pk_mul_f64", reinterpret_cast(inst), + selected_exec_fn(InstructionExecutionId::VPkMulF64Vop3p)), vdst(128, OperandType::OPR_VGPR, reinterpret_cast(inst)->vdst), src0(128, OperandType::OPR_SRC, reinterpret_cast(inst)->src0), src1(128, OperandType::OPR_SRC, reinterpret_cast(inst)->src1) { @@ -4228,13 +4232,15 @@ VPkMulF64Vop3p::VPkMulF64Vop3p(const MachineInst *inst) num_src_ = 2; num_dst_ = 1; if (reinterpret_cast(inst)->src0 == 255) - src0 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src0 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); if (reinterpret_cast(inst)->src1 == 255) - src1 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src1 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); vdst.set_vgpr_msb_role(amdgpu::VgprMsbRole::Dst); src0.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src0); src1.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src1); @@ -4248,6 +4254,75 @@ DecodeResult decodeVPkMulF64Vop3p(const MachineInst *opcode, const DecodeErrorEm emit_error, LiteralSupport::Literal32, 2); if (validation.failed()) [[unlikely]] return Result::failure(); + if (reinterpret_cast(inst)->opsel != 0u || + reinterpret_cast(inst)->opsel_hi != 3u || + reinterpret_cast(inst)->opsel_hi_2 != 1u) [[unlikely]] + return emit_error.emit() << "V_PK_MUL_F64 has an invalid packed F64 element layout"; + if (reinterpret_cast(inst)->src2 != 128u) [[unlikely]] + return emit_error.emit() << "V_PK_MUL_F64 has an invalid unused src2 encoding"; + if (reinterpret_cast(inst)->vdst > 252u) [[unlikely]] + return emit_error.emit() + << "V_PK_MUL_F64 has a vdst register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->vdst & 1u) != 0u) [[unlikely]] + return emit_error.emit() << "V_PK_MUL_F64 has an invalid vdst register tuple alignment"; + if ((reinterpret_cast(inst)->src0 <= 105u && + reinterpret_cast(inst)->src0 > 102u) || + (reinterpret_cast(inst)->src0 >= 256u && + reinterpret_cast(inst)->src0 > 508u)) [[unlikely]] + return emit_error.emit() + << "V_PK_MUL_F64 has a src0 register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->src0 <= 105u && + (reinterpret_cast(inst)->src0 % 4u) != 0u) || + (reinterpret_cast(inst)->src0 >= 108u && + reinterpret_cast(inst)->src0 <= 123u && + ((reinterpret_cast(inst)->src0 - 108u) % 4u) != 0u) || + (reinterpret_cast(inst)->src0 >= 256u && + (reinterpret_cast(inst)->src0 & 1u) != 0u)) [[unlikely]] + return emit_error.emit() << "V_PK_MUL_F64 has an invalid src0 register tuple alignment"; + if (!((reinterpret_cast(inst)->src0 <= 100u && + (reinterpret_cast(inst)->src0 % 4u) == 0u) || + (reinterpret_cast(inst)->src0 >= 108u && + reinterpret_cast(inst)->src0 <= 120u && + ((reinterpret_cast(inst)->src0 - 108u) % 4u) == 0u) || + reinterpret_cast(inst)->src0 == 124u || + (reinterpret_cast(inst)->src0 >= 128u && + reinterpret_cast(inst)->src0 <= 208u) || + (reinterpret_cast(inst)->src0 >= 240u && + reinterpret_cast(inst)->src0 <= 248u) || + reinterpret_cast(inst)->src0 == 255u || + (reinterpret_cast(inst)->src0 >= 256u && + reinterpret_cast(inst)->src0 <= 508u && + (reinterpret_cast(inst)->src0 & 1u) == 0u))) [[unlikely]] + return emit_error.emit() << "V_PK_MUL_F64 has an invalid src0 packed F64 source selector"; + if ((reinterpret_cast(inst)->src1 <= 105u && + reinterpret_cast(inst)->src1 > 102u) || + (reinterpret_cast(inst)->src1 >= 256u && + reinterpret_cast(inst)->src1 > 508u)) [[unlikely]] + return emit_error.emit() + << "V_PK_MUL_F64 has a src1 register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->src1 <= 105u && + (reinterpret_cast(inst)->src1 % 4u) != 0u) || + (reinterpret_cast(inst)->src1 >= 108u && + reinterpret_cast(inst)->src1 <= 123u && + ((reinterpret_cast(inst)->src1 - 108u) % 4u) != 0u) || + (reinterpret_cast(inst)->src1 >= 256u && + (reinterpret_cast(inst)->src1 & 1u) != 0u)) [[unlikely]] + return emit_error.emit() << "V_PK_MUL_F64 has an invalid src1 register tuple alignment"; + if (!((reinterpret_cast(inst)->src1 <= 100u && + (reinterpret_cast(inst)->src1 % 4u) == 0u) || + (reinterpret_cast(inst)->src1 >= 108u && + reinterpret_cast(inst)->src1 <= 120u && + ((reinterpret_cast(inst)->src1 - 108u) % 4u) == 0u) || + reinterpret_cast(inst)->src1 == 124u || + (reinterpret_cast(inst)->src1 >= 128u && + reinterpret_cast(inst)->src1 <= 208u) || + (reinterpret_cast(inst)->src1 >= 240u && + reinterpret_cast(inst)->src1 <= 248u) || + reinterpret_cast(inst)->src1 == 255u || + (reinterpret_cast(inst)->src1 >= 256u && + reinterpret_cast(inst)->src1 <= 508u && + (reinterpret_cast(inst)->src1 & 1u) == 0u))) [[unlikely]] + return emit_error.emit() << "V_PK_MUL_F64 has an invalid src1 packed F64 source selector"; if (reinterpret_cast(inst)->src0 == amdgpu::SRC_DPP || amdgpu::dpp::is_src_dpp8(reinterpret_cast(inst)->src0)) [[unlikely]] @@ -4257,7 +4332,8 @@ DecodeResult decodeVPkMulF64Vop3p(const MachineInst *opcode, const DecodeErrorEm } // namespace detail VPkAddF64Vop3p::VPkAddF64Vop3p(const MachineInst *inst) - : Vop3p("v_pk_add_f64", reinterpret_cast(inst), nullptr), + : Vop3p("v_pk_add_f64", reinterpret_cast(inst), + selected_exec_fn(InstructionExecutionId::VPkAddF64Vop3p)), vdst(128, OperandType::OPR_VGPR, reinterpret_cast(inst)->vdst), src0(128, OperandType::OPR_SRC, reinterpret_cast(inst)->src0), src1(128, OperandType::OPR_SRC, reinterpret_cast(inst)->src1) { @@ -4268,13 +4344,15 @@ VPkAddF64Vop3p::VPkAddF64Vop3p(const MachineInst *inst) num_src_ = 2; num_dst_ = 1; if (reinterpret_cast(inst)->src0 == 255) - src0 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src0 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); if (reinterpret_cast(inst)->src1 == 255) - src1 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src1 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); vdst.set_vgpr_msb_role(amdgpu::VgprMsbRole::Dst); src0.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src0); src1.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src1); @@ -4288,6 +4366,75 @@ DecodeResult decodeVPkAddF64Vop3p(const MachineInst *opcode, const DecodeErrorEm emit_error, LiteralSupport::Literal32, 2); if (validation.failed()) [[unlikely]] return Result::failure(); + if (reinterpret_cast(inst)->opsel != 0u || + reinterpret_cast(inst)->opsel_hi != 3u || + reinterpret_cast(inst)->opsel_hi_2 != 1u) [[unlikely]] + return emit_error.emit() << "V_PK_ADD_F64 has an invalid packed F64 element layout"; + if (reinterpret_cast(inst)->src2 != 128u) [[unlikely]] + return emit_error.emit() << "V_PK_ADD_F64 has an invalid unused src2 encoding"; + if (reinterpret_cast(inst)->vdst > 252u) [[unlikely]] + return emit_error.emit() + << "V_PK_ADD_F64 has a vdst register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->vdst & 1u) != 0u) [[unlikely]] + return emit_error.emit() << "V_PK_ADD_F64 has an invalid vdst register tuple alignment"; + if ((reinterpret_cast(inst)->src0 <= 105u && + reinterpret_cast(inst)->src0 > 102u) || + (reinterpret_cast(inst)->src0 >= 256u && + reinterpret_cast(inst)->src0 > 508u)) [[unlikely]] + return emit_error.emit() + << "V_PK_ADD_F64 has a src0 register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->src0 <= 105u && + (reinterpret_cast(inst)->src0 % 4u) != 0u) || + (reinterpret_cast(inst)->src0 >= 108u && + reinterpret_cast(inst)->src0 <= 123u && + ((reinterpret_cast(inst)->src0 - 108u) % 4u) != 0u) || + (reinterpret_cast(inst)->src0 >= 256u && + (reinterpret_cast(inst)->src0 & 1u) != 0u)) [[unlikely]] + return emit_error.emit() << "V_PK_ADD_F64 has an invalid src0 register tuple alignment"; + if (!((reinterpret_cast(inst)->src0 <= 100u && + (reinterpret_cast(inst)->src0 % 4u) == 0u) || + (reinterpret_cast(inst)->src0 >= 108u && + reinterpret_cast(inst)->src0 <= 120u && + ((reinterpret_cast(inst)->src0 - 108u) % 4u) == 0u) || + reinterpret_cast(inst)->src0 == 124u || + (reinterpret_cast(inst)->src0 >= 128u && + reinterpret_cast(inst)->src0 <= 208u) || + (reinterpret_cast(inst)->src0 >= 240u && + reinterpret_cast(inst)->src0 <= 248u) || + reinterpret_cast(inst)->src0 == 255u || + (reinterpret_cast(inst)->src0 >= 256u && + reinterpret_cast(inst)->src0 <= 508u && + (reinterpret_cast(inst)->src0 & 1u) == 0u))) [[unlikely]] + return emit_error.emit() << "V_PK_ADD_F64 has an invalid src0 packed F64 source selector"; + if ((reinterpret_cast(inst)->src1 <= 105u && + reinterpret_cast(inst)->src1 > 102u) || + (reinterpret_cast(inst)->src1 >= 256u && + reinterpret_cast(inst)->src1 > 508u)) [[unlikely]] + return emit_error.emit() + << "V_PK_ADD_F64 has a src1 register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->src1 <= 105u && + (reinterpret_cast(inst)->src1 % 4u) != 0u) || + (reinterpret_cast(inst)->src1 >= 108u && + reinterpret_cast(inst)->src1 <= 123u && + ((reinterpret_cast(inst)->src1 - 108u) % 4u) != 0u) || + (reinterpret_cast(inst)->src1 >= 256u && + (reinterpret_cast(inst)->src1 & 1u) != 0u)) [[unlikely]] + return emit_error.emit() << "V_PK_ADD_F64 has an invalid src1 register tuple alignment"; + if (!((reinterpret_cast(inst)->src1 <= 100u && + (reinterpret_cast(inst)->src1 % 4u) == 0u) || + (reinterpret_cast(inst)->src1 >= 108u && + reinterpret_cast(inst)->src1 <= 120u && + ((reinterpret_cast(inst)->src1 - 108u) % 4u) == 0u) || + reinterpret_cast(inst)->src1 == 124u || + (reinterpret_cast(inst)->src1 >= 128u && + reinterpret_cast(inst)->src1 <= 208u) || + (reinterpret_cast(inst)->src1 >= 240u && + reinterpret_cast(inst)->src1 <= 248u) || + reinterpret_cast(inst)->src1 == 255u || + (reinterpret_cast(inst)->src1 >= 256u && + reinterpret_cast(inst)->src1 <= 508u && + (reinterpret_cast(inst)->src1 & 1u) == 0u))) [[unlikely]] + return emit_error.emit() << "V_PK_ADD_F64 has an invalid src1 packed F64 source selector"; if (reinterpret_cast(inst)->src0 == amdgpu::SRC_DPP || amdgpu::dpp::is_src_dpp8(reinterpret_cast(inst)->src0)) [[unlikely]] @@ -4531,7 +4678,8 @@ DecodeResult decodeVPkSubNcU64Vop3p(const MachineInst *opcode, } // namespace detail VPkMaxNumF64Vop3p::VPkMaxNumF64Vop3p(const MachineInst *inst) - : Vop3p("v_pk_max_num_f64", reinterpret_cast(inst), nullptr), + : Vop3p("v_pk_max_num_f64", reinterpret_cast(inst), + selected_exec_fn(InstructionExecutionId::VPkMaxNumF64Vop3p)), vdst(128, OperandType::OPR_VGPR, reinterpret_cast(inst)->vdst), src0(128, OperandType::OPR_SRC, reinterpret_cast(inst)->src0), src1(128, OperandType::OPR_SRC, reinterpret_cast(inst)->src1) { @@ -4542,13 +4690,15 @@ VPkMaxNumF64Vop3p::VPkMaxNumF64Vop3p(const MachineInst *inst) num_src_ = 2; num_dst_ = 1; if (reinterpret_cast(inst)->src0 == 255) - src0 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src0 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); if (reinterpret_cast(inst)->src1 == 255) - src1 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src1 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); vdst.set_vgpr_msb_role(amdgpu::VgprMsbRole::Dst); src0.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src0); src1.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src1); @@ -4563,6 +4713,75 @@ DecodeResult decodeVPkMaxNumF64Vop3p(const MachineInst *opcode, emit_error, LiteralSupport::Literal32, 2); if (validation.failed()) [[unlikely]] return Result::failure(); + if (reinterpret_cast(inst)->opsel != 0u || + reinterpret_cast(inst)->opsel_hi != 3u || + reinterpret_cast(inst)->opsel_hi_2 != 1u) [[unlikely]] + return emit_error.emit() << "V_PK_MAX_NUM_F64 has an invalid packed F64 element layout"; + if (reinterpret_cast(inst)->src2 != 128u) [[unlikely]] + return emit_error.emit() << "V_PK_MAX_NUM_F64 has an invalid unused src2 encoding"; + if (reinterpret_cast(inst)->vdst > 252u) [[unlikely]] + return emit_error.emit() + << "V_PK_MAX_NUM_F64 has a vdst register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->vdst & 1u) != 0u) [[unlikely]] + return emit_error.emit() << "V_PK_MAX_NUM_F64 has an invalid vdst register tuple alignment"; + if ((reinterpret_cast(inst)->src0 <= 105u && + reinterpret_cast(inst)->src0 > 102u) || + (reinterpret_cast(inst)->src0 >= 256u && + reinterpret_cast(inst)->src0 > 508u)) [[unlikely]] + return emit_error.emit() + << "V_PK_MAX_NUM_F64 has a src0 register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->src0 <= 105u && + (reinterpret_cast(inst)->src0 % 4u) != 0u) || + (reinterpret_cast(inst)->src0 >= 108u && + reinterpret_cast(inst)->src0 <= 123u && + ((reinterpret_cast(inst)->src0 - 108u) % 4u) != 0u) || + (reinterpret_cast(inst)->src0 >= 256u && + (reinterpret_cast(inst)->src0 & 1u) != 0u)) [[unlikely]] + return emit_error.emit() << "V_PK_MAX_NUM_F64 has an invalid src0 register tuple alignment"; + if (!((reinterpret_cast(inst)->src0 <= 100u && + (reinterpret_cast(inst)->src0 % 4u) == 0u) || + (reinterpret_cast(inst)->src0 >= 108u && + reinterpret_cast(inst)->src0 <= 120u && + ((reinterpret_cast(inst)->src0 - 108u) % 4u) == 0u) || + reinterpret_cast(inst)->src0 == 124u || + (reinterpret_cast(inst)->src0 >= 128u && + reinterpret_cast(inst)->src0 <= 208u) || + (reinterpret_cast(inst)->src0 >= 240u && + reinterpret_cast(inst)->src0 <= 248u) || + reinterpret_cast(inst)->src0 == 255u || + (reinterpret_cast(inst)->src0 >= 256u && + reinterpret_cast(inst)->src0 <= 508u && + (reinterpret_cast(inst)->src0 & 1u) == 0u))) [[unlikely]] + return emit_error.emit() << "V_PK_MAX_NUM_F64 has an invalid src0 packed F64 source selector"; + if ((reinterpret_cast(inst)->src1 <= 105u && + reinterpret_cast(inst)->src1 > 102u) || + (reinterpret_cast(inst)->src1 >= 256u && + reinterpret_cast(inst)->src1 > 508u)) [[unlikely]] + return emit_error.emit() + << "V_PK_MAX_NUM_F64 has a src1 register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->src1 <= 105u && + (reinterpret_cast(inst)->src1 % 4u) != 0u) || + (reinterpret_cast(inst)->src1 >= 108u && + reinterpret_cast(inst)->src1 <= 123u && + ((reinterpret_cast(inst)->src1 - 108u) % 4u) != 0u) || + (reinterpret_cast(inst)->src1 >= 256u && + (reinterpret_cast(inst)->src1 & 1u) != 0u)) [[unlikely]] + return emit_error.emit() << "V_PK_MAX_NUM_F64 has an invalid src1 register tuple alignment"; + if (!((reinterpret_cast(inst)->src1 <= 100u && + (reinterpret_cast(inst)->src1 % 4u) == 0u) || + (reinterpret_cast(inst)->src1 >= 108u && + reinterpret_cast(inst)->src1 <= 120u && + ((reinterpret_cast(inst)->src1 - 108u) % 4u) == 0u) || + reinterpret_cast(inst)->src1 == 124u || + (reinterpret_cast(inst)->src1 >= 128u && + reinterpret_cast(inst)->src1 <= 208u) || + (reinterpret_cast(inst)->src1 >= 240u && + reinterpret_cast(inst)->src1 <= 248u) || + reinterpret_cast(inst)->src1 == 255u || + (reinterpret_cast(inst)->src1 >= 256u && + reinterpret_cast(inst)->src1 <= 508u && + (reinterpret_cast(inst)->src1 & 1u) == 0u))) [[unlikely]] + return emit_error.emit() << "V_PK_MAX_NUM_F64 has an invalid src1 packed F64 source selector"; if (reinterpret_cast(inst)->src0 == amdgpu::SRC_DPP || amdgpu::dpp::is_src_dpp8(reinterpret_cast(inst)->src0)) [[unlikely]] @@ -4572,7 +4791,8 @@ DecodeResult decodeVPkMaxNumF64Vop3p(const MachineInst *opcode, } // namespace detail VPkMinNumF64Vop3p::VPkMinNumF64Vop3p(const MachineInst *inst) - : Vop3p("v_pk_min_num_f64", reinterpret_cast(inst), nullptr), + : Vop3p("v_pk_min_num_f64", reinterpret_cast(inst), + selected_exec_fn(InstructionExecutionId::VPkMinNumF64Vop3p)), vdst(128, OperandType::OPR_VGPR, reinterpret_cast(inst)->vdst), src0(128, OperandType::OPR_SRC, reinterpret_cast(inst)->src0), src1(128, OperandType::OPR_SRC, reinterpret_cast(inst)->src1) { @@ -4583,13 +4803,15 @@ VPkMinNumF64Vop3p::VPkMinNumF64Vop3p(const MachineInst *inst) num_src_ = 2; num_dst_ = 1; if (reinterpret_cast(inst)->src0 == 255) - src0 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src0 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); if (reinterpret_cast(inst)->src1 == 255) - src1 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src1 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); vdst.set_vgpr_msb_role(amdgpu::VgprMsbRole::Dst); src0.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src0); src1.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src1); @@ -4604,6 +4826,75 @@ DecodeResult decodeVPkMinNumF64Vop3p(const MachineInst *opcode, emit_error, LiteralSupport::Literal32, 2); if (validation.failed()) [[unlikely]] return Result::failure(); + if (reinterpret_cast(inst)->opsel != 0u || + reinterpret_cast(inst)->opsel_hi != 3u || + reinterpret_cast(inst)->opsel_hi_2 != 1u) [[unlikely]] + return emit_error.emit() << "V_PK_MIN_NUM_F64 has an invalid packed F64 element layout"; + if (reinterpret_cast(inst)->src2 != 128u) [[unlikely]] + return emit_error.emit() << "V_PK_MIN_NUM_F64 has an invalid unused src2 encoding"; + if (reinterpret_cast(inst)->vdst > 252u) [[unlikely]] + return emit_error.emit() + << "V_PK_MIN_NUM_F64 has a vdst register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->vdst & 1u) != 0u) [[unlikely]] + return emit_error.emit() << "V_PK_MIN_NUM_F64 has an invalid vdst register tuple alignment"; + if ((reinterpret_cast(inst)->src0 <= 105u && + reinterpret_cast(inst)->src0 > 102u) || + (reinterpret_cast(inst)->src0 >= 256u && + reinterpret_cast(inst)->src0 > 508u)) [[unlikely]] + return emit_error.emit() + << "V_PK_MIN_NUM_F64 has a src0 register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->src0 <= 105u && + (reinterpret_cast(inst)->src0 % 4u) != 0u) || + (reinterpret_cast(inst)->src0 >= 108u && + reinterpret_cast(inst)->src0 <= 123u && + ((reinterpret_cast(inst)->src0 - 108u) % 4u) != 0u) || + (reinterpret_cast(inst)->src0 >= 256u && + (reinterpret_cast(inst)->src0 & 1u) != 0u)) [[unlikely]] + return emit_error.emit() << "V_PK_MIN_NUM_F64 has an invalid src0 register tuple alignment"; + if (!((reinterpret_cast(inst)->src0 <= 100u && + (reinterpret_cast(inst)->src0 % 4u) == 0u) || + (reinterpret_cast(inst)->src0 >= 108u && + reinterpret_cast(inst)->src0 <= 120u && + ((reinterpret_cast(inst)->src0 - 108u) % 4u) == 0u) || + reinterpret_cast(inst)->src0 == 124u || + (reinterpret_cast(inst)->src0 >= 128u && + reinterpret_cast(inst)->src0 <= 208u) || + (reinterpret_cast(inst)->src0 >= 240u && + reinterpret_cast(inst)->src0 <= 248u) || + reinterpret_cast(inst)->src0 == 255u || + (reinterpret_cast(inst)->src0 >= 256u && + reinterpret_cast(inst)->src0 <= 508u && + (reinterpret_cast(inst)->src0 & 1u) == 0u))) [[unlikely]] + return emit_error.emit() << "V_PK_MIN_NUM_F64 has an invalid src0 packed F64 source selector"; + if ((reinterpret_cast(inst)->src1 <= 105u && + reinterpret_cast(inst)->src1 > 102u) || + (reinterpret_cast(inst)->src1 >= 256u && + reinterpret_cast(inst)->src1 > 508u)) [[unlikely]] + return emit_error.emit() + << "V_PK_MIN_NUM_F64 has a src1 register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->src1 <= 105u && + (reinterpret_cast(inst)->src1 % 4u) != 0u) || + (reinterpret_cast(inst)->src1 >= 108u && + reinterpret_cast(inst)->src1 <= 123u && + ((reinterpret_cast(inst)->src1 - 108u) % 4u) != 0u) || + (reinterpret_cast(inst)->src1 >= 256u && + (reinterpret_cast(inst)->src1 & 1u) != 0u)) [[unlikely]] + return emit_error.emit() << "V_PK_MIN_NUM_F64 has an invalid src1 register tuple alignment"; + if (!((reinterpret_cast(inst)->src1 <= 100u && + (reinterpret_cast(inst)->src1 % 4u) == 0u) || + (reinterpret_cast(inst)->src1 >= 108u && + reinterpret_cast(inst)->src1 <= 120u && + ((reinterpret_cast(inst)->src1 - 108u) % 4u) == 0u) || + reinterpret_cast(inst)->src1 == 124u || + (reinterpret_cast(inst)->src1 >= 128u && + reinterpret_cast(inst)->src1 <= 208u) || + (reinterpret_cast(inst)->src1 >= 240u && + reinterpret_cast(inst)->src1 <= 248u) || + reinterpret_cast(inst)->src1 == 255u || + (reinterpret_cast(inst)->src1 >= 256u && + reinterpret_cast(inst)->src1 <= 508u && + (reinterpret_cast(inst)->src1 & 1u) == 0u))) [[unlikely]] + return emit_error.emit() << "V_PK_MIN_NUM_F64 has an invalid src1 packed F64 source selector"; if (reinterpret_cast(inst)->src0 == amdgpu::SRC_DPP || amdgpu::dpp::is_src_dpp8(reinterpret_cast(inst)->src0)) [[unlikely]] diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.h index e9ecdd97dd9c..621a37962fd7 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.h @@ -944,6 +944,7 @@ class VPkFmaF64Vop3p : public Vop3p { class VPkMulF64Vop3p : public Vop3p { public: VPkMulF64Vop3p(const MachineInst *inst); + void execute_impl(amdgpu::Wavefront &wf); Operand vdst; Operand src0; Operand src1; @@ -952,6 +953,7 @@ class VPkMulF64Vop3p : public Vop3p { class VPkAddF64Vop3p : public Vop3p { public: VPkAddF64Vop3p(const MachineInst *inst); + void execute_impl(amdgpu::Wavefront &wf); Operand vdst; Operand src0; Operand src1; @@ -978,6 +980,7 @@ class VPkSubNcU64Vop3p : public Vop3p { class VPkMaxNumF64Vop3p : public Vop3p { public: VPkMaxNumF64Vop3p(const MachineInst *inst); + void execute_impl(amdgpu::Wavefront &wf); Operand vdst; Operand src0; Operand src1; @@ -986,6 +989,7 @@ class VPkMaxNumF64Vop3p : public Vop3p { class VPkMinNumF64Vop3p : public Vop3p { public: VPkMinNumF64Vop3p(const MachineInst *inst); + void execute_impl(amdgpu::Wavefront &wf); Operand vdst; Operand src0; Operand src1; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp index d0e92992b0d9..582f22f2242a 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp @@ -2918,6 +2918,58 @@ void VWmmaF3232x16x128F4Vop3p::execute_impl(amdgpu::Wavefront &wf) { amdgpu::wmma_c_modifier(inst_.neg, inst_.neg_hi)); } +void VPkMulF64Vop3p::execute_impl(amdgpu::Wavefront &wf) { + uint64_t exec = wf.exec(); + for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { + if (!(exec & (1ULL << lane))) + continue; + const auto lhs = read_pk_u64_pair(src0, wf, lane); + const auto rhs = read_pk_u64_pair(src1, wf, lane); + constexpr uint64_t kSignBit = 0x8000000000000000ULL; + const auto apply = [&](uint64_t lhs_bits, uint64_t rhs_bits, bool negate_lhs, bool negate_rhs) { + if (negate_lhs) + lhs_bits ^= kSignBit; + if (negate_rhs) + rhs_bits ^= kSignBit; + uint64_t result = + amdgpu::fp_mode::binary_f64(lhs_bits, rhs_bits, amdgpu::fp_mode::BinaryF64Op::Multiply, + wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64()); + return amdgpu::fp_mode::finish_f64(result, wf.fp_round_mode_f16_f64(), 0, inst_.clamp, + amdgpu::floating_clamp_nan_to_zero(wf)); + }; + const uint64_t result_lo = apply(lhs.lo, rhs.lo, (inst_.neg & 1u) != 0, (inst_.neg & 2u) != 0); + const uint64_t result_hi = + apply(lhs.hi, rhs.hi, (inst_.neg_hi & 1u) != 0, (inst_.neg_hi & 2u) != 0); + write_pk_u64_pair(vdst, wf, lane, {result_lo, result_hi}); + } +} + +void VPkAddF64Vop3p::execute_impl(amdgpu::Wavefront &wf) { + uint64_t exec = wf.exec(); + for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { + if (!(exec & (1ULL << lane))) + continue; + const auto lhs = read_pk_u64_pair(src0, wf, lane); + const auto rhs = read_pk_u64_pair(src1, wf, lane); + constexpr uint64_t kSignBit = 0x8000000000000000ULL; + const auto apply = [&](uint64_t lhs_bits, uint64_t rhs_bits, bool negate_lhs, bool negate_rhs) { + if (negate_lhs) + lhs_bits ^= kSignBit; + if (negate_rhs) + rhs_bits ^= kSignBit; + uint64_t result = + amdgpu::fp_mode::binary_f64(lhs_bits, rhs_bits, amdgpu::fp_mode::BinaryF64Op::Add, + wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64()); + return amdgpu::fp_mode::finish_f64(result, wf.fp_round_mode_f16_f64(), 0, inst_.clamp, + amdgpu::floating_clamp_nan_to_zero(wf)); + }; + const uint64_t result_lo = apply(lhs.lo, rhs.lo, (inst_.neg & 1u) != 0, (inst_.neg & 2u) != 0); + const uint64_t result_hi = + apply(lhs.hi, rhs.hi, (inst_.neg_hi & 1u) != 0, (inst_.neg_hi & 2u) != 0); + write_pk_u64_pair(vdst, wf, lane, {result_lo, result_hi}); + } +} + void VPkAddNcU64Vop3p::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2984,6 +3036,58 @@ void VPkSubNcU64Vop3p::execute_impl(amdgpu::Wavefront &wf) { } } +void VPkMaxNumF64Vop3p::execute_impl(amdgpu::Wavefront &wf) { + uint64_t exec = wf.exec(); + for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { + if (!(exec & (1ULL << lane))) + continue; + const auto lhs = read_pk_u64_pair(src0, wf, lane); + const auto rhs = read_pk_u64_pair(src1, wf, lane); + constexpr uint64_t kSignBit = 0x8000000000000000ULL; + const auto apply = [&](uint64_t lhs_bits, uint64_t rhs_bits, bool negate_lhs, bool negate_rhs) { + if (negate_lhs) + lhs_bits ^= kSignBit; + if (negate_rhs) + rhs_bits ^= kSignBit; + uint64_t result = amdgpu::fp_mode::binary_f64( + lhs_bits, rhs_bits, amdgpu::fp_mode::BinaryF64Op::MaximumNumber, + wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64()); + return amdgpu::fp_mode::finish_f64(result, wf.fp_round_mode_f16_f64(), 0, inst_.clamp, + amdgpu::floating_clamp_nan_to_zero(wf)); + }; + const uint64_t result_lo = apply(lhs.lo, rhs.lo, (inst_.neg & 1u) != 0, (inst_.neg & 2u) != 0); + const uint64_t result_hi = + apply(lhs.hi, rhs.hi, (inst_.neg_hi & 1u) != 0, (inst_.neg_hi & 2u) != 0); + write_pk_u64_pair(vdst, wf, lane, {result_lo, result_hi}); + } +} + +void VPkMinNumF64Vop3p::execute_impl(amdgpu::Wavefront &wf) { + uint64_t exec = wf.exec(); + for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { + if (!(exec & (1ULL << lane))) + continue; + const auto lhs = read_pk_u64_pair(src0, wf, lane); + const auto rhs = read_pk_u64_pair(src1, wf, lane); + constexpr uint64_t kSignBit = 0x8000000000000000ULL; + const auto apply = [&](uint64_t lhs_bits, uint64_t rhs_bits, bool negate_lhs, bool negate_rhs) { + if (negate_lhs) + lhs_bits ^= kSignBit; + if (negate_rhs) + rhs_bits ^= kSignBit; + uint64_t result = amdgpu::fp_mode::binary_f64( + lhs_bits, rhs_bits, amdgpu::fp_mode::BinaryF64Op::MinimumNumber, + wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64()); + return amdgpu::fp_mode::finish_f64(result, wf.fp_round_mode_f16_f64(), 0, inst_.clamp, + amdgpu::floating_clamp_nan_to_zero(wf)); + }; + const uint64_t result_lo = apply(lhs.lo, rhs.lo, (inst_.neg & 1u) != 0, (inst_.neg & 2u) != 0); + const uint64_t result_hi = + apply(lhs.hi, rhs.hi, (inst_.neg_hi & 1u) != 0, (inst_.neg_hi & 2u) != 0); + write_pk_u64_pair(vdst, wf, lane, {result_lo, result_hi}); + } +} + void VPkLshlAddU64Vop3p::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); std::array results{}; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h index 2d4036ed1f68..e97979493405 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h @@ -670,6 +670,48 @@ inline uint64_t fma_f64(uint64_t src0, uint64_t src1, uint64_t src2, uint32_t ro return result; } +/// @brief Binary F64 operations implemented by the shared MODE-aware helper. +enum class BinaryF64Op { Add, Multiply, MaximumNumber, MinimumNumber }; + +/// @brief Execute a binary F64 operation under MODE.FP_ROUND and MODE.FP_DENORM. +/// @details MaximumNumber and MinimumNumber return the numeric operand for a +/// one-NaN input and explicitly select +0/-0 for signed-zero ties respectively. +inline uint64_t binary_f64(uint64_t src0, uint64_t src1, BinaryF64Op operation, uint32_t round_mode, + uint32_t denorm_mode) { + if ((denorm_mode & 1u) == 0) { + src0 = detail::flush_f64(src0); + src1 = detail::flush_f64(src1); + } + + uint64_t result; + { + detail::ScopedFenv environment(round_mode); + const double lhs = std::bit_cast(src0); + const double rhs = std::bit_cast(src1); + const double value = [&] { + switch (operation) { + case BinaryF64Op::Add: + return lhs + rhs; + case BinaryF64Op::Multiply: + return lhs * rhs; + case BinaryF64Op::MaximumNumber: + if ((src0 & 0x7fffffffffffffffULL) == 0 && (src1 & 0x7fffffffffffffffULL) == 0) + return std::bit_cast((src0 & src1) & 0x8000000000000000ULL); + return std::fmax(lhs, rhs); + case BinaryF64Op::MinimumNumber: + if ((src0 & 0x7fffffffffffffffULL) == 0 && (src1 & 0x7fffffffffffffffULL) == 0) + return std::bit_cast((src0 | src1) & 0x8000000000000000ULL); + return std::fmin(lhs, rhs); + } + return std::numeric_limits::quiet_NaN(); + }(); + result = std::bit_cast(value); + } + if ((denorm_mode & 2u) == 0) + result = detail::flush_f64(result); + return result; +} + /// @brief Apply F64 OMOD/CLAMP under the architectural rounding mode. /// @details A nonzero OMOD flushes a denormal result and converts either signed zero to +0. inline uint64_t finish_f64(uint64_t value, uint32_t round_mode, uint32_t omod, bool clamp, diff --git a/emulation/rocjitsu/tests/cdna5_execution_test.cpp b/emulation/rocjitsu/tests/cdna5_execution_test.cpp index 0c70c2e1f27c..2f76436714f5 100644 --- a/emulation/rocjitsu/tests/cdna5_execution_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_execution_test.cpp @@ -715,6 +715,8 @@ TEST(FpModePolicyTest, F64HelpersRestoreAmbientHostEnvironment) { for (const uint64_t input : {kOne, kInfinity, kQuietNan}) { (void)amdgpu::fp_mode::fma_f64(input, kOne, kOne, 1, 3); EXPECT_EQ(std::fegetround(), FE_DOWNWARD); + (void)amdgpu::fp_mode::binary_f64(input, kOne, amdgpu::fp_mode::BinaryF64Op::Add, 1, 3); + EXPECT_EQ(std::fegetround(), FE_DOWNWARD); (void)amdgpu::fp_mode::finish_f64(input, 1, 1, false, true); EXPECT_EQ(std::fegetround(), FE_DOWNWARD); } @@ -1774,6 +1776,835 @@ void write_sgpr_packed_u64(amdgpu::ComputeUnitCore &cu, amdgpu::Wavefront &wf, u } } +TEST(Gfx1251PackedF64ExecutionTest, BasicArithmeticHasExecutionCallbacks) { + // Exact public LLVM gfx1251_asm_vop3p.s encodings for the VGPR forms. + constexpr std::array, 4> kWords{{ + {0xCC4B4004u, 0x1A021908u}, // v_pk_add_f64 v[4:7], v[8:11], v[12:15] + {0xCC3C4004u, 0x1A021908u}, // v_pk_mul_f64 v[4:7], v[8:11], v[12:15] + {0xCC4E4004u, 0x1A021908u}, // v_pk_max_num_f64 v[4:7], v[8:11], v[12:15] + {0xCC4F4004u, 0x1A021908u}, // v_pk_min_num_f64 v[4:7], v[8:11], v[12:15] + }}; + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + + for (const auto &words : kWords) { + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + EXPECT_NE(decoded->execute, nullptr) << decoded->mnemonic(); + } +} + +TEST(Gfx1251PackedF64ExecutionTest, LaterExecutionSlicesRemainCallbackFree) { + // Exact public LLVM encodings for the two instructions intentionally left + // model-only by this slice: gfx1251_asm_vop3p.s and + // gfx1251_asm_wmma_w32.s. + constexpr std::array, 2> kWords{{ + {0xCC3B4004u, 0x1C421908u}, // v_pk_fma_f64 v[4:7], v[8:11], v[12:15], v[16:19] + {0xCC5B0008u, 0x1C220900u}, // v_wmma_f64_16x16x4_f64 v[8:15], v[0:1], v[2:3], v[4:7] + }}; + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + + for (const auto &words : kWords) { + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + EXPECT_EQ(decoded->execute, nullptr) << decoded->mnemonic(); + } +} + +TEST(Gfx1251PackedF64ExecutionTest, BasicArithmeticExecutesBothElementsAndHonorsExec) { + struct Operation { + std::string_view name; + std::array words; + PackedU64Pair expected; + }; + constexpr auto bits = [](double value) { return std::bit_cast(value); }; + constexpr std::array kOperations{ + Operation{"add", {0xCC4B4004u, 0x1A021908u}, {bits(7.0), bits(1.0)}}, + Operation{"mul", {0xCC3C4004u, 0x1A021908u}, {bits(10.0), bits(-12.0)}}, + Operation{"max-num", {0xCC4E4004u, 0x1A021908u}, {bits(5.0), bits(4.0)}}, + Operation{"min-num", {0xCC4F4004u, 0x1A021908u}, {bits(2.0), bits(-3.0)}}, + }; + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(0x5u); + constexpr PackedU64Pair kLhs{bits(2.0), bits(-3.0)}; + constexpr PackedU64Pair kRhs{bits(5.0), bits(4.0)}; + constexpr PackedU64Pair kInactive{0xdeadbeefcafef00dULL, 0xbaadf00d12345678ULL}; + + for (const auto &operation : kOperations) { + SCOPED_TRACE(operation.name); + for (uint32_t lane = 0; lane < 3; ++lane) { + write_vgpr_packed_u64(*cu, *wf, 8, lane, kLhs); + write_vgpr_packed_u64(*cu, *wf, 12, lane, kRhs); + write_vgpr_packed_u64(*cu, *wf, 4, lane, kInactive); + } + std::unique_ptr decoded(decode_valid(*decoder, operation.words.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + cu->execute_instruction(decoded.get(), *wf); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), operation.expected); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 1), kInactive); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 2), operation.expected); + } +} + +TEST(Gfx1251PackedF64ExecutionTest, ExecutesWave32Lane31WithoutChangingVccOrInactiveLanes) { + constexpr auto bits = [](double value) { return std::bit_cast(value); }; + constexpr std::array kAdd{0xCC4B4004u, 0x1A021908u}; + constexpr PackedU64Pair kInactiveSeed{0x0123456789abcdefULL, 0xfedcba9876543210ULL}; + constexpr uint64_t kVccSeed = 0x5a5aa5a5deadbeefULL; + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + ASSERT_EQ(wf->wf_size(), 32u); + wf->set_exec_raw(uint64_t{1} << 31); + wf->set_vcc_raw(kVccSeed); + write_vgpr_packed_u64(*cu, *wf, 8, 31, {bits(2.0), bits(-3.0)}); + write_vgpr_packed_u64(*cu, *wf, 12, 31, {bits(5.0), bits(4.0)}); + write_vgpr_packed_u64(*cu, *wf, 4, 0, kInactiveSeed); + + std::unique_ptr decoded(decode_valid(*decoder, kAdd.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + cu->execute_instruction(decoded.get(), *wf); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 31), (PackedU64Pair{bits(7.0), bits(1.0)})); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), kInactiveSeed); + EXPECT_EQ(wf->vcc(), kVccSeed); +} + +TEST(Gfx1251PackedF64ExecutionTest, ExecZeroPreservesDestinationAndStatus) { + constexpr std::array kAdd{0xCC4B4004u, 0x1A021908u}; + constexpr PackedU64Pair kDestinationSeed{0x0123456789abcdefULL, 0xfedcba9876543210ULL}; + constexpr uint64_t kVccSeed = 0x5a5aa5a5deadbeefULL; + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec_raw(0); + wf->set_vcc_raw(kVccSeed); + wf->write_scc(true); + write_vgpr_packed_u64(*cu, *wf, 4, 0, kDestinationSeed); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {0x3ff0000000000000ULL, 0x4000000000000000ULL}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {0x4008000000000000ULL, 0x4010000000000000ULL}); + + std::unique_ptr decoded(decode_valid(*decoder, kAdd.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + cu->execute_instruction(decoded.get(), *wf); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), kDestinationSeed); + EXPECT_EQ(wf->vcc(), kVccSeed); + EXPECT_TRUE(wf->read_scc()); +} + +TEST(Gfx1251PackedF64ExecutionTest, BasicArithmeticExecutesEveryPublicLlvmSourceForm) { + struct SourceForm { + std::string_view name; + uint32_t second_word; + uint32_t literal; + PackedU64Pair lhs; + PackedU64Pair rhs; + }; + constexpr auto bits = [](double value) { return std::bit_cast(value); }; + constexpr std::array kForms{ + SourceForm{"vgpr-vgpr", 0x1A021908u, 0u, {bits(10.0), bits(20.0)}, {bits(3.0), bits(4.0)}}, + SourceForm{"sgpr-sgpr", 0x1A001808u, 0u, {bits(100.0), bits(100.0)}, {bits(7.0), bits(7.0)}}, + SourceForm{"vgpr-sgpr", 0x1A001908u, 0u, {bits(10.0), bits(20.0)}, {bits(7.0), bits(7.0)}}, + SourceForm{"sgpr-vgpr", 0x1A021808u, 0u, {bits(100.0), bits(100.0)}, {bits(3.0), bits(4.0)}}, + SourceForm{"vgpr-null", 0x1A00F908u, 0u, {bits(10.0), bits(20.0)}, {0u, 0u}}, + SourceForm{ + "vgpr-inline-one", 0x1A01E508u, 0u, {bits(10.0), bits(20.0)}, {bits(1.0), bits(1.0)}}, + SourceForm{ + "inline-one-vgpr", 0x1A0210F2u, 0u, {bits(1.0), bits(1.0)}, {bits(10.0), bits(20.0)}}, + SourceForm{"literal-vgpr", + 0x1A0210FFu, + 0x40594000u, + {bits(101.0), bits(101.0)}, + {bits(10.0), bits(20.0)}}, + SourceForm{"vgpr-literal", + 0x1A01FF08u, + 0x40594000u, + {bits(10.0), bits(20.0)}, + {bits(101.0), bits(101.0)}}, + SourceForm{"shared-literal", + 0x1A01FEFFu, + 0x40594000u, + {bits(101.0), bits(101.0)}, + {bits(101.0), bits(101.0)}}, + }; + struct Operation { + std::string_view name; + uint32_t first_word; + }; + constexpr std::array kOperations{ + Operation{"add", 0xCC4B4004u}, + Operation{"mul", 0xCC3C4004u}, + Operation{"max-num", 0xCC4E4004u}, + Operation{"min-num", 0xCC4F4004u}, + }; + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {bits(10.0), bits(20.0)}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {bits(3.0), bits(4.0)}); + // Packed-FP64 SGPR operands broadcast their low 64 bits. Keep different + // high words in the backing tuple so the result proves they are ignored. + write_sgpr_packed_u64(*cu, *wf, 8, {bits(100.0), bits(200.0)}); + write_sgpr_packed_u64(*cu, *wf, 12, {bits(7.0), bits(8.0)}); + + const auto reference = [](std::string_view operation, double lhs, double rhs) { + if (operation == "add") + return lhs + rhs; + if (operation == "mul") + return lhs * rhs; + if (operation == "max-num") + return std::fmax(lhs, rhs); + return std::fmin(lhs, rhs); + }; + for (const auto &operation : kOperations) { + for (const auto &form : kForms) { + SCOPED_TRACE(operation.name); + SCOPED_TRACE(form.name); + const std::array words{operation.first_word, form.second_word, form.literal}; + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + cu->execute_instruction(decoded.get(), *wf); + const PackedU64Pair expected{ + std::bit_cast(reference(operation.name, std::bit_cast(form.lhs[0]), + std::bit_cast(form.rhs[0]))), + std::bit_cast(reference(operation.name, std::bit_cast(form.lhs[1]), + std::bit_cast(form.rhs[1]))), + }; + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), expected); + if (form.literal != 0u) { + for (uint32_t source = 0; source < 2; ++source) { + if (form.name != "shared-literal" && source != (form.name == "literal-vgpr" ? 0u : 1u)) + continue; + const Operand *literal_operand = decoded->src_operand(source); + ASSERT_NE(literal_operand, nullptr); + EXPECT_EQ(amdgpu::RegisterAccess(*wf).read_lane64(*literal_operand, 0), bits(101.0)); + } + } + } + } +} + +TEST(Gfx1251PackedF64ExecutionTest, ExecutesInlineScalarBoundarySources) { + constexpr auto bits = [](double value) { return std::bit_cast(value); }; + struct InlineCase { + std::string_view name; + uint16_t opcode; + uint16_t src0; + uint16_t src1; + uint64_t expected; + }; + constexpr std::array kCases{ + InlineCase{"positive-integer-low", cdna5::kVPkAddF64Vop3p, 128, 124, 0}, + InlineCase{"positive-integer-high", cdna5::kVPkAddF64Vop3p, 192, 124, 64}, + InlineCase{"negative-integer-low", cdna5::kVPkMaxNumF64Vop3p, 193, 242, bits(1.0)}, + InlineCase{"negative-integer-high", cdna5::kVPkMaxNumF64Vop3p, 208, 242, bits(1.0)}, + InlineCase{"float-half", cdna5::kVPkAddF64Vop3p, 240, 124, bits(0.5)}, + InlineCase{"inverse-two-pi", cdna5::kVPkAddF64Vop3p, 248, 124, 0x3FC45F306DC9C882ULL}, + }; + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + wf->set_mode_raw(3u << 6); + wf->write_scc(true); + + for (const auto &test_case : kCases) { + SCOPED_TRACE(test_case.name); + const auto words = cdna5::build_vop3p(test_case.opcode, {.vdst = 4, + .opsel_hi_2 = 1, + .src0 = test_case.src0, + .src1 = test_case.src1, + .src2 = 128, + .opsel_hi = 3}); + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + cu->execute_instruction(decoded.get(), *wf); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), + (PackedU64Pair{test_case.expected, test_case.expected})); + } + EXPECT_TRUE(wf->read_scc()); +} + +TEST(Gfx1251PackedF64ExecutionTest, ExecutesFinalRegisterTupleBoundaries) { + constexpr auto bits = [](double value) { return std::bit_cast(value); }; + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + wf->set_mode_raw(3u << 6); + + const auto execute_add = [&](cdna5::Vop3pBuilderFields fields, PackedU64Pair expected) { + const auto words = cdna5::build_vop3p(cdna5::kVPkAddF64Vop3p, fields); + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + cu->execute_instruction(decoded.get(), *wf); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, fields.vdst, 0), expected); + }; + + write_sgpr_packed_u64(*cu, *wf, 100, {bits(2.0), bits(99.0)}); + execute_add({.vdst = 4, .opsel_hi_2 = 1, .src0 = 100, .src1 = 124, .src2 = 128, .opsel_hi = 3}, + {bits(2.0), bits(2.0)}); + + wf->set_ttmp(12, static_cast(bits(3.0))); + wf->set_ttmp(13, static_cast(bits(3.0) >> 32)); + wf->set_ttmp(14, static_cast(bits(99.0))); + wf->set_ttmp(15, static_cast(bits(99.0) >> 32)); + execute_add({.vdst = 4, .opsel_hi_2 = 1, .src0 = 120, .src1 = 124, .src2 = 128, .opsel_hi = 3}, + {bits(3.0), bits(3.0)}); + + write_vgpr_packed_u64(*cu, *wf, 252, 0, {bits(4.0), bits(5.0)}); + execute_add({.vdst = 252, .opsel_hi_2 = 1, .src0 = 508, .src1 = 100, .src2 = 128, .opsel_hi = 3}, + {bits(6.0), bits(7.0)}); +} + +TEST(Gfx1251PackedF64ExecutionTest, PublicModifiersClampAndOverlappingDestinationAreSafe) { + constexpr auto bits = [](double value) { return std::bit_cast(value); }; + struct Operation { + std::string_view name; + uint32_t base_first_word; + PackedU64Pair neg_src0_expected; + PackedU64Pair neg_src1_expected; + PackedU64Pair clamp_expected; + PackedU64Pair combined_expected; + }; + constexpr std::array kOperations{ + Operation{"add", + 0xCC4B4004u, + {bits(3.0), bits(7.0)}, + {bits(-3.0), bits(-7.0)}, + {bits(0.0), bits(1.0)}, + {bits(1.0), bits(1.0)}}, + Operation{"mul", + 0xCC3C4004u, + {bits(-10.0), bits(12.0)}, + {bits(-10.0), bits(12.0)}, + {bits(0.0), bits(1.0)}, + {bits(0.0), bits(1.0)}}, + Operation{"max-num", + 0xCC4E4004u, + {bits(5.0), bits(4.0)}, + {bits(2.0), bits(-3.0)}, + {bits(1.0), bits(1.0)}, + {bits(1.0), bits(1.0)}}, + Operation{"min-num", + 0xCC4F4004u, + {bits(-2.0), bits(3.0)}, + {bits(-5.0), bits(-4.0)}, + {bits(0.0), bits(1.0)}, + {bits(0.0), bits(1.0)}}, + }; + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + + for (const auto &operation : kOperations) { + SCOPED_TRACE(operation.name); + const std::array modifier_cases{ + std::pair{std::array{operation.base_first_word + 0x100u, 0x3A021908u}, + operation.neg_src0_expected}, + std::pair{std::array{operation.base_first_word + 0x200u, 0x5A021908u}, + operation.neg_src1_expected}, + }; + for (const auto &[words, expected] : modifier_cases) { + write_vgpr_packed_u64(*cu, *wf, 8, 0, {bits(2.0), bits(-3.0)}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {bits(5.0), bits(4.0)}); + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + cu->execute_instruction(decoded.get(), *wf); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), expected); + } + + const std::array clamp_words{operation.base_first_word + 0x8000u, 0x1A021908u}; + write_vgpr_packed_u64(*cu, *wf, 8, 0, {bits(-2.0), bits(2.0)}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {bits(1.0), bits(2.0)}); + std::unique_ptr clamp(decode_valid(*decoder, clamp_words.data())); + ASSERT_NE(clamp, nullptr); + ASSERT_NE(clamp->execute, nullptr); + cu->execute_instruction(clamp.get(), *wf); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), operation.clamp_expected); + + const auto combined_words = + cdna5::build_vop3p((operation.base_first_word >> 16) & 0x7fu, {.vdst = 4, + .neg_hi = 1, + .opsel_hi_2 = 1, + .clamp = 1, + .src0 = 264, + .src1 = 268, + .src2 = 128, + .opsel_hi = 3, + .neg = 1}); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {bits(2.0), bits(-3.0)}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {bits(5.0), bits(4.0)}); + std::unique_ptr combined(decode_valid(*decoder, combined_words.data())); + ASSERT_NE(combined, nullptr); + ASSERT_NE(combined->execute, nullptr); + cu->execute_instruction(combined.get(), *wf); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), operation.combined_expected); + } + + const std::array asymmetric_cases{ + std::pair{cdna5::Vop3pBuilderFields{.vdst = 4, + .opsel_hi_2 = 1, + .src0 = 264, + .src1 = 268, + .src2 = 128, + .opsel_hi = 3, + .neg = 1}, + PackedU64Pair{bits(3.0), bits(1.0)}}, + std::pair{cdna5::Vop3pBuilderFields{.vdst = 4, + .neg_hi = 2, + .opsel_hi_2 = 1, + .src0 = 264, + .src1 = 268, + .src2 = 128, + .opsel_hi = 3}, + PackedU64Pair{bits(7.0), bits(-7.0)}}, + }; + for (const auto &[fields, expected] : asymmetric_cases) { + const auto words = cdna5::build_vop3p(cdna5::kVPkAddF64Vop3p, fields); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {bits(2.0), bits(-3.0)}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {bits(5.0), bits(4.0)}); + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + cu->execute_instruction(decoded.get(), *wf); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), expected); + } + + constexpr uint64_t kQuietNan = 0x7ff8000000001234ULL; + constexpr uint64_t kSignalingNan = 0x7ff0000000000001ULL; + for (const uint16_t opcode : {cdna5::kVPkAddF64Vop3p, cdna5::kVPkMulF64Vop3p, + cdna5::kVPkMaxNumF64Vop3p, cdna5::kVPkMinNumF64Vop3p}) { + const auto clamp_nan_words = cdna5::build_vop3p(opcode, {.vdst = 4, + .opsel_hi_2 = 1, + .clamp = 1, + .src0 = 264, + .src1 = 124, + .src2 = 128, + .opsel_hi = 3}); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {kQuietNan, kSignalingNan}); + std::unique_ptr clamp_nan(decode_valid(*decoder, clamp_nan_words.data())); + ASSERT_NE(clamp_nan, nullptr); + ASSERT_NE(clamp_nan->execute, nullptr); + cu->execute_instruction(clamp_nan.get(), *wf); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), (PackedU64Pair{bits(0.0), bits(0.0)})); + } + + struct OverlapCase { + std::string_view name; + uint8_t vdst; + }; + constexpr std::array kOverlapCases{ + OverlapCase{"exact-src0", 8}, + OverlapCase{"exact-src1", 12}, + OverlapCase{"partial-both", 10}, + }; + for (const auto &test_case : kOverlapCases) { + SCOPED_TRACE(test_case.name); + const auto overlap_words = cdna5::build_vop3p(cdna5::kVPkAddF64Vop3p, {.vdst = test_case.vdst, + .opsel_hi_2 = 1, + .src0 = 264, + .src1 = 268, + .src2 = 128, + .opsel_hi = 3}); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {bits(2.0), bits(-3.0)}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {bits(5.0), bits(4.0)}); + std::unique_ptr overlap(decode_valid(*decoder, overlap_words.data())); + ASSERT_NE(overlap, nullptr); + ASSERT_NE(overlap->execute, nullptr); + cu->execute_instruction(overlap.get(), *wf); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, test_case.vdst, 0), + (PackedU64Pair{bits(7.0), bits(1.0)})); + } +} + +TEST(Gfx1251PackedF64ExecutionTest, AddAndMultiplyHonorRoundingAndDenormModes) { + constexpr auto bits = [](double value) { return std::bit_cast(value); }; + const auto add_words = cdna5::build_vop3p( + cdna5::kVPkAddF64Vop3p, + {.vdst = 4, .opsel_hi_2 = 1, .src0 = 264, .src1 = 268, .src2 = 128, .opsel_hi = 3}); + const auto mul_words = cdna5::build_vop3p( + cdna5::kVPkMulF64Vop3p, + {.vdst = 4, .opsel_hi_2 = 1, .src0 = 264, .src1 = 268, .src2 = 128, .opsel_hi = 3}); + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + std::unique_ptr add(decode_valid(*decoder, add_words.data())); + std::unique_ptr mul(decode_valid(*decoder, mul_words.data())); + ASSERT_NE(add, nullptr); + ASSERT_NE(mul, nullptr); + ASSERT_NE(add->execute, nullptr); + ASSERT_NE(mul->execute, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + + constexpr uint64_t kOne = bits(1.0); + constexpr uint64_t kNegativeOne = bits(-1.0); + constexpr uint64_t kHalfUlpAtOne = bits(0x1p-53); + constexpr uint64_t kNegativeHalfUlpAtOne = bits(-0x1p-53); + constexpr std::array kAddExpected{{ + {kOne, kNegativeOne}, + {kOne + 1, kNegativeOne}, + {kOne, kNegativeOne + 1}, + {kOne, kNegativeOne}, + }}; + for (uint32_t round = 0; round < 4; ++round) { + wf->set_mode_raw((round << 2) | (3u << 6)); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {kOne, kNegativeOne}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {kHalfUlpAtOne, kNegativeHalfUlpAtOne}); + cu->execute_instruction(add.get(), *wf); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), kAddExpected[round]) << round; + } + + constexpr uint64_t kNextOne = kOne + 1; + constexpr uint64_t kRoundedSquare = kOne + 2; + constexpr std::array kMulExpected{kRoundedSquare, kRoundedSquare + 1, kRoundedSquare, + kRoundedSquare}; + for (uint32_t round = 0; round < 4; ++round) { + wf->set_mode_raw((round << 2) | (3u << 6)); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {kNextOne, kNextOne}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {kNextOne, kNextOne}); + cu->execute_instruction(mul.get(), *wf); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), + (PackedU64Pair{kMulExpected[round], kMulExpected[round]})) + << round; + } + + constexpr uint64_t kMinSubnormal = 1u; + constexpr uint64_t kHalfMinNormal = 0x0008000000000000ULL; + constexpr uint64_t kMinNormal = 0x0010000000000000ULL; + constexpr std::array kAddDenormExpected{0u, 0u, 0u, kMinSubnormal}; + constexpr std::array kMulDenormExpected{0u, 0u, kHalfMinNormal, kHalfMinNormal}; + for (uint32_t denorm = 0; denorm < 4; ++denorm) { + wf->set_mode_raw(denorm << 6); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {kMinSubnormal, kMinNormal}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {0u, bits(0.5)}); + cu->execute_instruction(add.get(), *wf); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0)[0], kAddDenormExpected[denorm]) << denorm; + cu->execute_instruction(mul.get(), *wf); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0)[1], kMulDenormExpected[denorm]) << denorm; + } +} + +TEST(Gfx1251PackedF64ExecutionTest, MinMaxNumberHonorEveryDenormMode) { + constexpr uint64_t kPositiveZero = 0; + constexpr uint64_t kNegativeZero = 0x8000000000000000ULL; + constexpr uint64_t kMinSubnormal = 1; + constexpr uint64_t kNegativeMinSubnormal = kNegativeZero | kMinSubnormal; + constexpr std::array kMaxExpected{kPositiveZero, kPositiveZero, kPositiveZero, + kMinSubnormal}; + constexpr std::array kMinExpected{kNegativeZero, kNegativeZero, kNegativeZero, + kNegativeMinSubnormal}; + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + const auto max_words = cdna5::build_vop3p( + cdna5::kVPkMaxNumF64Vop3p, + {.vdst = 4, .opsel_hi_2 = 1, .src0 = 264, .src1 = 268, .src2 = 128, .opsel_hi = 3}); + const auto min_words = cdna5::build_vop3p( + cdna5::kVPkMinNumF64Vop3p, + {.vdst = 4, .opsel_hi_2 = 1, .src0 = 264, .src1 = 268, .src2 = 128, .opsel_hi = 3}); + std::unique_ptr maximum(decode_valid(*decoder, max_words.data())); + std::unique_ptr minimum(decode_valid(*decoder, min_words.data())); + ASSERT_NE(maximum, nullptr); + ASSERT_NE(minimum, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + + for (uint32_t denorm = 0; denorm < 4; ++denorm) { + wf->set_mode_raw(denorm << 6); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {kMinSubnormal, kNegativeMinSubnormal}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {kPositiveZero, kPositiveZero}); + cu->execute_instruction(maximum.get(), *wf); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0)[0], kMaxExpected[denorm]) << denorm; + cu->execute_instruction(minimum.get(), *wf); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0)[1], kMinExpected[denorm]) << denorm; + } +} + +TEST(Gfx1251PackedF64ExecutionTest, AddAndMultiplyHandleNanInfinityAndSignedZero) { + constexpr auto bits = [](double value) { return std::bit_cast(value); }; + constexpr uint64_t kPositiveZero = 0u; + constexpr uint64_t kNegativeZero = 0x8000000000000000ULL; + constexpr uint64_t kPositiveInfinity = bits(std::numeric_limits::infinity()); + constexpr uint64_t kNegativeInfinity = bits(-std::numeric_limits::infinity()); + constexpr std::array, 2> kCases{{ + {cdna5::kVPkAddF64Vop3p, {kNegativeInfinity, kNegativeZero}}, + {cdna5::kVPkMulF64Vop3p, {kPositiveZero, bits(2.0)}}, + }}; + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + wf->set_mode_raw(3u << 6); + + for (const auto &[opcode, rhs] : kCases) { + const auto words = cdna5::build_vop3p( + opcode, {.vdst = 4, .opsel_hi_2 = 1, .src0 = 264, .src1 = 268, .src2 = 128, .opsel_hi = 3}); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {kPositiveInfinity, kNegativeZero}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, rhs); + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + cu->execute_instruction(decoded.get(), *wf); + const PackedU64Pair result = read_vgpr_packed_u64(*cu, *wf, 4, 0); + EXPECT_TRUE(std::isnan(std::bit_cast(result[0]))); + EXPECT_EQ(result[1], kNegativeZero); + } +} + +TEST(Gfx1251PackedF64ExecutionTest, MinMaxNumberHandleNanInfinityAndSignedZero) { + constexpr auto bits = [](double value) { return std::bit_cast(value); }; + constexpr uint64_t kQuietNan = 0x7ff8000000001234ULL; + constexpr uint64_t kSignalingNan = 0x7ff0000000000001ULL; + constexpr uint64_t kPositiveZero = 0u; + constexpr uint64_t kNegativeZero = 0x8000000000000000ULL; + constexpr uint64_t kPositiveInfinity = bits(std::numeric_limits::infinity()); + constexpr uint64_t kNegativeInfinity = bits(-std::numeric_limits::infinity()); + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + wf->set_mode_raw(3u << 6); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {kQuietNan, kNegativeZero}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {bits(2.0), kPositiveZero}); + + for (const auto &[words, expected] : {std::pair{std::array{0xCC4E4004u, 0x1A021908u}, + PackedU64Pair{bits(2.0), kPositiveZero}}, + std::pair{std::array{0xCC4F4004u, 0x1A021908u}, + PackedU64Pair{bits(2.0), kNegativeZero}}}) { + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + cu->execute_instruction(decoded.get(), *wf); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), expected); + } + + write_vgpr_packed_u64(*cu, *wf, 8, 0, {kQuietNan, bits(3.0)}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {bits(2.0), kSignalingNan}); + for (const auto &words : {std::array{0xCC4E4004u, 0x1A021908u}, + std::array{0xCC4F4004u, 0x1A021908u}}) { + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + cu->execute_instruction(decoded.get(), *wf); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), (PackedU64Pair{bits(2.0), bits(3.0)})); + } + + struct NanCase { + std::string_view name; + uint64_t lhs; + uint64_t rhs; + uint64_t expected; + bool expect_nan; + }; + constexpr std::array kNanCases{ + NanCase{"qnan-left", kQuietNan, bits(2.0), bits(2.0), false}, + NanCase{"qnan-right", bits(2.0), kQuietNan, bits(2.0), false}, + NanCase{"snan-left", kSignalingNan, bits(3.0), bits(3.0), false}, + NanCase{"snan-right", bits(3.0), kSignalingNan, bits(3.0), false}, + NanCase{"qnan-snan", kQuietNan, kSignalingNan, 0, true}, + NanCase{"snan-qnan", kSignalingNan, kQuietNan, 0, true}, + }; + for (const auto &words : {std::array{0xCC4E4004u, 0x1A021908u}, + std::array{0xCC4F4004u, 0x1A021908u}}) { + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + for (const auto &test_case : kNanCases) { + SCOPED_TRACE(decoded->mnemonic()); + SCOPED_TRACE(test_case.name); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {test_case.lhs, test_case.lhs}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {test_case.rhs, test_case.rhs}); + cu->execute_instruction(decoded.get(), *wf); + const PackedU64Pair result = read_vgpr_packed_u64(*cu, *wf, 4, 0); + if (test_case.expect_nan) { + EXPECT_TRUE(std::isnan(std::bit_cast(result[0]))); + EXPECT_TRUE(std::isnan(std::bit_cast(result[1]))); + } else { + EXPECT_EQ(result, (PackedU64Pair{test_case.expected, test_case.expected})); + } + } + } + + write_vgpr_packed_u64(*cu, *wf, 8, 0, {kPositiveZero, kNegativeZero}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {kNegativeZero, kPositiveZero}); + for (const auto &[words, expected] : {std::pair{std::array{0xCC4E4004u, 0x1A021908u}, + PackedU64Pair{kPositiveZero, kPositiveZero}}, + std::pair{std::array{0xCC4F4004u, 0x1A021908u}, + PackedU64Pair{kNegativeZero, kNegativeZero}}}) { + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + cu->execute_instruction(decoded.get(), *wf); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), expected); + } + + write_vgpr_packed_u64(*cu, *wf, 8, 0, {kNegativeInfinity, kQuietNan}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {kPositiveInfinity, kQuietNan}); + for (const auto &words : {std::array{0xCC4E4004u, 0x1A021908u}, + std::array{0xCC4F4004u, 0x1A021908u}}) { + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + cu->execute_instruction(decoded.get(), *wf); + const PackedU64Pair result = read_vgpr_packed_u64(*cu, *wf, 4, 0); + EXPECT_EQ(result[0], decoded->mnemonic() == std::string_view("v_pk_max_num_f64") + ? kPositiveInfinity + : kNegativeInfinity); + EXPECT_TRUE(std::isnan(std::bit_cast(result[1]))); + } +} + +TEST(Gfx1251PackedF64ExecutionTest, RejectsUndefinedLayoutsAndOutOfRangeRegisterTuples) { + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + constexpr std::array kOpcodes{ + cdna5::kVPkAddF64Vop3p, + cdna5::kVPkMulF64Vop3p, + cdna5::kVPkMaxNumF64Vop3p, + cdna5::kVPkMinNumF64Vop3p, + }; + constexpr std::array kInvalidFields{ + cdna5::Vop3pBuilderFields{.vdst = 4, + .opsel = 1, + .opsel_hi_2 = 1, + .src0 = 264, + .src1 = 268, + .src2 = 128, + .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{ + .vdst = 4, .opsel_hi_2 = 0, .src0 = 264, .src1 = 268, .src2 = 128, .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{ + .vdst = 4, .opsel_hi_2 = 1, .src0 = 264, .src1 = 268, .src2 = 129, .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{ + .vdst = 253, .opsel_hi_2 = 1, .src0 = 264, .src1 = 268, .src2 = 128, .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{ + .vdst = 4, .opsel_hi_2 = 1, .src0 = 509, .src1 = 268, .src2 = 128, .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{ + .vdst = 4, .opsel_hi_2 = 1, .src0 = 264, .src1 = 104, .src2 = 128, .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{ + .vdst = 3, .opsel_hi_2 = 1, .src0 = 264, .src1 = 268, .src2 = 128, .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{ + .vdst = 4, .opsel_hi_2 = 1, .src0 = 265, .src1 = 268, .src2 = 128, .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{ + .vdst = 4, .opsel_hi_2 = 1, .src0 = 10, .src1 = 268, .src2 = 128, .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{ + .vdst = 4, .opsel_hi_2 = 1, .src0 = 109, .src1 = 268, .src2 = 128, .opsel_hi = 3}, + }; + for (const uint16_t opcode : kOpcodes) { + SCOPED_TRACE(opcode); + for (const auto &fields : kInvalidFields) { + const auto words = cdna5::build_vop3p(opcode, fields); + EXPECT_EQ(decode_valid(*decoder, words.data()), nullptr); + } + } + + constexpr std::array kInvalidV2F64Selectors{ + 106, 107, 125, 126, 127, 209, 229, 230, 231, 232, 234, + 235, 236, 237, 238, 239, 249, 251, 252, 253, 254, + }; + for (const uint16_t opcode : kOpcodes) { + SCOPED_TRACE(opcode); + for (const uint16_t selector : kInvalidV2F64Selectors) { + for (const bool use_src1 : {false, true}) { + SCOPED_TRACE(selector); + SCOPED_TRACE(use_src1); + const auto words = cdna5::build_vop3p(opcode, {.vdst = 4, + .opsel_hi_2 = 1, + .src0 = use_src1 ? uint16_t{264} : selector, + .src1 = use_src1 ? selector : uint16_t{268}, + .src2 = 128, + .opsel_hi = 3}); + EXPECT_EQ(decode_valid(*decoder, words.data()), nullptr); + } + } + } + + constexpr std::array kValidV2F64Selectors{ + 124, 128, 208, 240, 248, 255, + }; + for (const uint16_t opcode : kOpcodes) { + SCOPED_TRACE(opcode); + for (const uint16_t selector : kValidV2F64Selectors) { + SCOPED_TRACE(selector); + const auto encoding = cdna5::build_vop3p( + opcode, + {.vdst = 4, .opsel_hi_2 = 1, .src0 = selector, .src1 = 268, .src2 = 128, .opsel_hi = 3}); + const std::array words{encoding[0], encoding[1], 0x65u}; + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + EXPECT_NE(decoded, nullptr); + } + } + + constexpr std::array kValidBoundaries{ + cdna5::Vop3pBuilderFields{ + .vdst = 252, .opsel_hi_2 = 1, .src0 = 508, .src1 = 100, .src2 = 128, .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{ + .vdst = 2, .opsel_hi_2 = 1, .src0 = 258, .src1 = 262, .src2 = 128, .opsel_hi = 3}, + }; + for (const uint16_t opcode : kOpcodes) { + SCOPED_TRACE(opcode); + for (const auto &fields : kValidBoundaries) { + const auto words = cdna5::build_vop3p(opcode, fields); + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + EXPECT_NE(decoded, nullptr); + } + } +} + TEST(Gfx1251PackedU64ExecutionTest, AddAndSubtractHaveExecutionCallbacks) { // Exact public LLVM gfx1251_asm_vop3p.s encodings for the VGPR forms. constexpr std::array, 2> kWords{{ diff --git a/shared/machine-readable-isa/isa/amdgpu_isa_cdna5_variants.json b/shared/machine-readable-isa/isa/amdgpu_isa_cdna5_variants.json index b9a63c93bfdb..d93b357026e6 100644 --- a/shared/machine-readable-isa/isa/amdgpu_isa_cdna5_variants.json +++ b/shared/machine-readable-isa/isa/amdgpu_isa_cdna5_variants.json @@ -46,10 +46,6 @@ ], "model_only_instructions": [ "V_PK_FMA_F64", - "V_PK_MUL_F64", - "V_PK_ADD_F64", - "V_PK_MAX_NUM_F64", - "V_PK_MIN_NUM_F64", "V_WMMA_F64_16X16X4_F64" ], "encodings": [ From fa3caccfdbd6bf843b476dedbf212cafdd0388cf Mon Sep 17 00:00:00 2001 From: hanhanW Date: Wed, 2 Sep 2026 08:51:28 -0700 Subject: [PATCH 2/8] [rocjitsu] Use std::isnan for packed FP64 minmax --- .../src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h index e97979493405..8dde5e5390b5 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h @@ -45,6 +45,8 @@ inline uint64_t flush_f64(uint64_t value) { return value; } +inline uint64_t quiet_nan_f64(uint64_t value) { return value | 0x0008000000000000ULL; } + inline int host_round_mode(uint32_t round_mode) { switch (round_mode & 3u) { case 1: @@ -695,10 +697,18 @@ inline uint64_t binary_f64(uint64_t src0, uint64_t src1, BinaryF64Op operation, case BinaryF64Op::Multiply: return lhs * rhs; case BinaryF64Op::MaximumNumber: + if (std::isnan(lhs)) + return std::bit_cast(std::isnan(rhs) ? detail::quiet_nan_f64(src0) : src1); + if (std::isnan(rhs)) + return lhs; if ((src0 & 0x7fffffffffffffffULL) == 0 && (src1 & 0x7fffffffffffffffULL) == 0) return std::bit_cast((src0 & src1) & 0x8000000000000000ULL); return std::fmax(lhs, rhs); case BinaryF64Op::MinimumNumber: + if (std::isnan(lhs)) + return std::bit_cast(std::isnan(rhs) ? detail::quiet_nan_f64(src0) : src1); + if (std::isnan(rhs)) + return lhs; if ((src0 & 0x7fffffffffffffffULL) == 0 && (src1 & 0x7fffffffffffffffULL) == 0) return std::bit_cast((src0 | src1) & 0x8000000000000000ULL); return std::fmin(lhs, rhs); From 5c28e95951a01dbbbb053cd8ac82aa2d35219e2f Mon Sep 17 00:00:00 2001 From: hanhanW Date: Wed, 2 Sep 2026 09:48:28 -0700 Subject: [PATCH 3/8] [rocjitsu] Use standard quiet NaN for packed FP64 minmax --- .../rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h index 8dde5e5390b5..ea172504f9c1 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h @@ -45,8 +45,6 @@ inline uint64_t flush_f64(uint64_t value) { return value; } -inline uint64_t quiet_nan_f64(uint64_t value) { return value | 0x0008000000000000ULL; } - inline int host_round_mode(uint32_t round_mode) { switch (round_mode & 3u) { case 1: @@ -698,7 +696,7 @@ inline uint64_t binary_f64(uint64_t src0, uint64_t src1, BinaryF64Op operation, return lhs * rhs; case BinaryF64Op::MaximumNumber: if (std::isnan(lhs)) - return std::bit_cast(std::isnan(rhs) ? detail::quiet_nan_f64(src0) : src1); + return std::isnan(rhs) ? std::numeric_limits::quiet_NaN() : rhs; if (std::isnan(rhs)) return lhs; if ((src0 & 0x7fffffffffffffffULL) == 0 && (src1 & 0x7fffffffffffffffULL) == 0) @@ -706,7 +704,7 @@ inline uint64_t binary_f64(uint64_t src0, uint64_t src1, BinaryF64Op operation, return std::fmax(lhs, rhs); case BinaryF64Op::MinimumNumber: if (std::isnan(lhs)) - return std::bit_cast(std::isnan(rhs) ? detail::quiet_nan_f64(src0) : src1); + return std::isnan(rhs) ? std::numeric_limits::quiet_NaN() : rhs; if (std::isnan(rhs)) return lhs; if ((src0 & 0x7fffffffffffffffULL) == 0 && (src1 & 0x7fffffffffffffffULL) == 0) From 8ff24446fb9e0357b406466f88ddb1551811ab36 Mon Sep 17 00:00:00 2001 From: hanhanW Date: Tue, 15 Sep 2026 09:47:57 -0700 Subject: [PATCH 4/8] address comments Signed-off-by: hanhanW --- .../python/amdisa/codegen/execute/__init__.py | 2 +- .../rocjitsu/tests/cdna5_execution_test.cpp | 67 +++++++------------ 2 files changed, 25 insertions(+), 44 deletions(-) diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py index 1c8407d47519..4bab3c3dce05 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py @@ -118,9 +118,9 @@ def _register_handlers() -> None: gen_pk_fmac_vop2, gen_pk_fmac_vop3, gen_pk_binop_f32, - gen_pk_binop_f64, gen_pk_ternary_f32, gen_pk_binop_u64, + gen_pk_binop_f64, gen_pk_lshl_add_u64, gen_pk_mov_b32, gen_mad_mix_f32, diff --git a/emulation/rocjitsu/tests/cdna5_execution_test.cpp b/emulation/rocjitsu/tests/cdna5_execution_test.cpp index 2f76436714f5..290bc2d70cf9 100644 --- a/emulation/rocjitsu/tests/cdna5_execution_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_execution_test.cpp @@ -1776,25 +1776,6 @@ void write_sgpr_packed_u64(amdgpu::ComputeUnitCore &cu, amdgpu::Wavefront &wf, u } } -TEST(Gfx1251PackedF64ExecutionTest, BasicArithmeticHasExecutionCallbacks) { - // Exact public LLVM gfx1251_asm_vop3p.s encodings for the VGPR forms. - constexpr std::array, 4> kWords{{ - {0xCC4B4004u, 0x1A021908u}, // v_pk_add_f64 v[4:7], v[8:11], v[12:15] - {0xCC3C4004u, 0x1A021908u}, // v_pk_mul_f64 v[4:7], v[8:11], v[12:15] - {0xCC4E4004u, 0x1A021908u}, // v_pk_max_num_f64 v[4:7], v[8:11], v[12:15] - {0xCC4F4004u, 0x1A021908u}, // v_pk_min_num_f64 v[4:7], v[8:11], v[12:15] - }}; - auto decoder = - make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); - ASSERT_NE(decoder, nullptr); - - for (const auto &words : kWords) { - std::unique_ptr decoded(decode_valid(*decoder, words.data())); - ASSERT_NE(decoded, nullptr); - EXPECT_NE(decoded->execute, nullptr) << decoded->mnemonic(); - } -} - TEST(Gfx1251PackedF64ExecutionTest, LaterExecutionSlicesRemainCallbackFree) { // Exact public LLVM encodings for the two instructions intentionally left // model-only by this slice: gfx1251_asm_vop3p.s and @@ -1849,7 +1830,7 @@ TEST(Gfx1251PackedF64ExecutionTest, BasicArithmeticExecutesBothElementsAndHonors std::unique_ptr decoded(decode_valid(*decoder, operation.words.data())); ASSERT_NE(decoded, nullptr); ASSERT_NE(decoded->execute, nullptr); - cu->execute_instruction(decoded.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), operation.expected); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 1), kInactive); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 2), operation.expected); @@ -1878,7 +1859,7 @@ TEST(Gfx1251PackedF64ExecutionTest, ExecutesWave32Lane31WithoutChangingVccOrInac std::unique_ptr decoded(decode_valid(*decoder, kAdd.data())); ASSERT_NE(decoded, nullptr); ASSERT_NE(decoded->execute, nullptr); - cu->execute_instruction(decoded.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 31), (PackedU64Pair{bits(7.0), bits(1.0)})); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), kInactiveSeed); EXPECT_EQ(wf->vcc(), kVccSeed); @@ -1905,7 +1886,7 @@ TEST(Gfx1251PackedF64ExecutionTest, ExecZeroPreservesDestinationAndStatus) { std::unique_ptr decoded(decode_valid(*decoder, kAdd.data())); ASSERT_NE(decoded, nullptr); ASSERT_NE(decoded->execute, nullptr); - cu->execute_instruction(decoded.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), kDestinationSeed); EXPECT_EQ(wf->vcc(), kVccSeed); EXPECT_TRUE(wf->read_scc()); @@ -1988,7 +1969,7 @@ TEST(Gfx1251PackedF64ExecutionTest, BasicArithmeticExecutesEveryPublicLlvmSource std::unique_ptr decoded(decode_valid(*decoder, words.data())); ASSERT_NE(decoded, nullptr); ASSERT_NE(decoded->execute, nullptr); - cu->execute_instruction(decoded.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); const PackedU64Pair expected{ std::bit_cast(reference(operation.name, std::bit_cast(form.lhs[0]), std::bit_cast(form.rhs[0]))), @@ -2048,7 +2029,7 @@ TEST(Gfx1251PackedF64ExecutionTest, ExecutesInlineScalarBoundarySources) { std::unique_ptr decoded(decode_valid(*decoder, words.data())); ASSERT_NE(decoded, nullptr); ASSERT_NE(decoded->execute, nullptr); - cu->execute_instruction(decoded.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), (PackedU64Pair{test_case.expected, test_case.expected})); } @@ -2072,7 +2053,7 @@ TEST(Gfx1251PackedF64ExecutionTest, ExecutesFinalRegisterTupleBoundaries) { std::unique_ptr decoded(decode_valid(*decoder, words.data())); ASSERT_NE(decoded, nullptr); ASSERT_NE(decoded->execute, nullptr); - cu->execute_instruction(decoded.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, fields.vdst, 0), expected); }; @@ -2151,7 +2132,7 @@ TEST(Gfx1251PackedF64ExecutionTest, PublicModifiersClampAndOverlappingDestinatio std::unique_ptr decoded(decode_valid(*decoder, words.data())); ASSERT_NE(decoded, nullptr); ASSERT_NE(decoded->execute, nullptr); - cu->execute_instruction(decoded.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), expected); } @@ -2161,7 +2142,7 @@ TEST(Gfx1251PackedF64ExecutionTest, PublicModifiersClampAndOverlappingDestinatio std::unique_ptr clamp(decode_valid(*decoder, clamp_words.data())); ASSERT_NE(clamp, nullptr); ASSERT_NE(clamp->execute, nullptr); - cu->execute_instruction(clamp.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(clamp.get(), *wf).succeeded()); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), operation.clamp_expected); const auto combined_words = @@ -2179,7 +2160,7 @@ TEST(Gfx1251PackedF64ExecutionTest, PublicModifiersClampAndOverlappingDestinatio std::unique_ptr combined(decode_valid(*decoder, combined_words.data())); ASSERT_NE(combined, nullptr); ASSERT_NE(combined->execute, nullptr); - cu->execute_instruction(combined.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(combined.get(), *wf).succeeded()); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), operation.combined_expected); } @@ -2208,7 +2189,7 @@ TEST(Gfx1251PackedF64ExecutionTest, PublicModifiersClampAndOverlappingDestinatio std::unique_ptr decoded(decode_valid(*decoder, words.data())); ASSERT_NE(decoded, nullptr); ASSERT_NE(decoded->execute, nullptr); - cu->execute_instruction(decoded.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), expected); } @@ -2227,7 +2208,7 @@ TEST(Gfx1251PackedF64ExecutionTest, PublicModifiersClampAndOverlappingDestinatio std::unique_ptr clamp_nan(decode_valid(*decoder, clamp_nan_words.data())); ASSERT_NE(clamp_nan, nullptr); ASSERT_NE(clamp_nan->execute, nullptr); - cu->execute_instruction(clamp_nan.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(clamp_nan.get(), *wf).succeeded()); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), (PackedU64Pair{bits(0.0), bits(0.0)})); } @@ -2253,7 +2234,7 @@ TEST(Gfx1251PackedF64ExecutionTest, PublicModifiersClampAndOverlappingDestinatio std::unique_ptr overlap(decode_valid(*decoder, overlap_words.data())); ASSERT_NE(overlap, nullptr); ASSERT_NE(overlap->execute, nullptr); - cu->execute_instruction(overlap.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(overlap.get(), *wf).succeeded()); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, test_case.vdst, 0), (PackedU64Pair{bits(7.0), bits(1.0)})); } @@ -2296,7 +2277,7 @@ TEST(Gfx1251PackedF64ExecutionTest, AddAndMultiplyHonorRoundingAndDenormModes) { wf->set_mode_raw((round << 2) | (3u << 6)); write_vgpr_packed_u64(*cu, *wf, 8, 0, {kOne, kNegativeOne}); write_vgpr_packed_u64(*cu, *wf, 12, 0, {kHalfUlpAtOne, kNegativeHalfUlpAtOne}); - cu->execute_instruction(add.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(add.get(), *wf).succeeded()); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), kAddExpected[round]) << round; } @@ -2308,7 +2289,7 @@ TEST(Gfx1251PackedF64ExecutionTest, AddAndMultiplyHonorRoundingAndDenormModes) { wf->set_mode_raw((round << 2) | (3u << 6)); write_vgpr_packed_u64(*cu, *wf, 8, 0, {kNextOne, kNextOne}); write_vgpr_packed_u64(*cu, *wf, 12, 0, {kNextOne, kNextOne}); - cu->execute_instruction(mul.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(mul.get(), *wf).succeeded()); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), (PackedU64Pair{kMulExpected[round], kMulExpected[round]})) << round; @@ -2323,9 +2304,9 @@ TEST(Gfx1251PackedF64ExecutionTest, AddAndMultiplyHonorRoundingAndDenormModes) { wf->set_mode_raw(denorm << 6); write_vgpr_packed_u64(*cu, *wf, 8, 0, {kMinSubnormal, kMinNormal}); write_vgpr_packed_u64(*cu, *wf, 12, 0, {0u, bits(0.5)}); - cu->execute_instruction(add.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(add.get(), *wf).succeeded()); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0)[0], kAddDenormExpected[denorm]) << denorm; - cu->execute_instruction(mul.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(mul.get(), *wf).succeeded()); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0)[1], kMulDenormExpected[denorm]) << denorm; } } @@ -2362,9 +2343,9 @@ TEST(Gfx1251PackedF64ExecutionTest, MinMaxNumberHonorEveryDenormMode) { wf->set_mode_raw(denorm << 6); write_vgpr_packed_u64(*cu, *wf, 8, 0, {kMinSubnormal, kNegativeMinSubnormal}); write_vgpr_packed_u64(*cu, *wf, 12, 0, {kPositiveZero, kPositiveZero}); - cu->execute_instruction(maximum.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(maximum.get(), *wf).succeeded()); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0)[0], kMaxExpected[denorm]) << denorm; - cu->execute_instruction(minimum.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(minimum.get(), *wf).succeeded()); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0)[1], kMinExpected[denorm]) << denorm; } } @@ -2397,7 +2378,7 @@ TEST(Gfx1251PackedF64ExecutionTest, AddAndMultiplyHandleNanInfinityAndSignedZero std::unique_ptr decoded(decode_valid(*decoder, words.data())); ASSERT_NE(decoded, nullptr); ASSERT_NE(decoded->execute, nullptr); - cu->execute_instruction(decoded.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); const PackedU64Pair result = read_vgpr_packed_u64(*cu, *wf, 4, 0); EXPECT_TRUE(std::isnan(std::bit_cast(result[0]))); EXPECT_EQ(result[1], kNegativeZero); @@ -2431,7 +2412,7 @@ TEST(Gfx1251PackedF64ExecutionTest, MinMaxNumberHandleNanInfinityAndSignedZero) std::unique_ptr decoded(decode_valid(*decoder, words.data())); ASSERT_NE(decoded, nullptr); ASSERT_NE(decoded->execute, nullptr); - cu->execute_instruction(decoded.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), expected); } @@ -2441,7 +2422,7 @@ TEST(Gfx1251PackedF64ExecutionTest, MinMaxNumberHandleNanInfinityAndSignedZero) std::array{0xCC4F4004u, 0x1A021908u}}) { std::unique_ptr decoded(decode_valid(*decoder, words.data())); ASSERT_NE(decoded, nullptr); - cu->execute_instruction(decoded.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), (PackedU64Pair{bits(2.0), bits(3.0)})); } @@ -2469,7 +2450,7 @@ TEST(Gfx1251PackedF64ExecutionTest, MinMaxNumberHandleNanInfinityAndSignedZero) SCOPED_TRACE(test_case.name); write_vgpr_packed_u64(*cu, *wf, 8, 0, {test_case.lhs, test_case.lhs}); write_vgpr_packed_u64(*cu, *wf, 12, 0, {test_case.rhs, test_case.rhs}); - cu->execute_instruction(decoded.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); const PackedU64Pair result = read_vgpr_packed_u64(*cu, *wf, 4, 0); if (test_case.expect_nan) { EXPECT_TRUE(std::isnan(std::bit_cast(result[0]))); @@ -2488,7 +2469,7 @@ TEST(Gfx1251PackedF64ExecutionTest, MinMaxNumberHandleNanInfinityAndSignedZero) PackedU64Pair{kNegativeZero, kNegativeZero}}}) { std::unique_ptr decoded(decode_valid(*decoder, words.data())); ASSERT_NE(decoded, nullptr); - cu->execute_instruction(decoded.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), expected); } @@ -2498,7 +2479,7 @@ TEST(Gfx1251PackedF64ExecutionTest, MinMaxNumberHandleNanInfinityAndSignedZero) std::array{0xCC4F4004u, 0x1A021908u}}) { std::unique_ptr decoded(decode_valid(*decoder, words.data())); ASSERT_NE(decoded, nullptr); - cu->execute_instruction(decoded.get(), *wf); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); const PackedU64Pair result = read_vgpr_packed_u64(*cu, *wf, 4, 0); EXPECT_EQ(result[0], decoded->mnemonic() == std::string_view("v_pk_max_num_f64") ? kPositiveInfinity From 723279e114436d3142874d70a2166d04e67c8fb0 Mon Sep 17 00:00:00 2001 From: hanhanW Date: Tue, 15 Sep 2026 10:32:12 -0700 Subject: [PATCH 5/8] [rocjitsu] Keep SGPR104 selector U64-only Signed-off-by: hanhanW --- .../lib/python/amdisa/codegen/_generator.py | 13 +++++++++++-- .../amdisa/tests/test_generator_profile_gates.py | 2 ++ .../isa/arch/amdgpu/generated/cdna5/vop3p.cpp | 16 ++++++++++++---- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py index 5452c96beb2f..39402165a60e 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py @@ -10416,6 +10416,10 @@ def gen_insts(self) -> None: if inst_sem.semantic_class == 'pk_binop_f64' else 'packed U64' ) + allows_sgpr104_128 = inst_sem.semantic_class in { + 'pk_binop_u64', + 'pk_lshl_add_u64', + } factory_validation_parts.append( f'if ({raw_inst}->opsel != 0u || ' f'{raw_inst}->opsel_hi != 3u || ' @@ -10457,7 +10461,7 @@ def gen_insts(self) -> None: max_sgpr = 106 - register_count max_vgpr = 512 - register_count invalid_sgpr_span = f'{raw_value} > {max_sgpr}u' - if opnd.size == 128: + if opnd.size == 128 and allows_sgpr104_128: # SGPR104_128 is a named VS_128 member: # s104:s105 provide the scalar U64 value, # while vcc_lo:vcc_hi complete the tuple. @@ -10489,9 +10493,14 @@ def gen_insts(self) -> None: ) if opnd.operand_type == 'OPR_SRC': if opnd.size == 128: + sgpr104_selector = ( + f'{raw_value} == 104u || ' + if allows_sgpr104_128 + else '' + ) valid_width_specific = ( f'({raw_value} <= 100u && ({raw_value} % 4u) == 0u) || ' - f'{raw_value} == 104u || ' + f'{sgpr104_selector}' f'({raw_value} >= 108u && {raw_value} <= 120u && ' f'(({raw_value} - 108u) % 4u) == 0u) || ' f'{raw_value} == 124u || ' diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py index ef4424d0a2ba..f4d3f532017c 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py @@ -5285,6 +5285,8 @@ def test_gfx1251_packed_f64_decode_rejects_undefined_layouts_and_register_tuples assert 'invalid src0 packed F64 source selector' in body assert 'invalid src1 packed F64 source selector' in body for operand_name in ('src0', 'src1'): + assert f'{operand_name} != 104u' not in body + assert f'{operand_name} == 104u' not in body assert f'{operand_name} <= 100u' in body assert f'{operand_name} >= 108u' in body assert f'{operand_name} <= 120u' in body diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.cpp index 3be41e86ae7c..cd710e396e49 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.cpp @@ -4258,7 +4258,9 @@ DecodeResult decodeVPkMulF64Vop3p(const MachineInst *opcode, const DecodeErrorEm reinterpret_cast(inst)->opsel_hi != 3u || reinterpret_cast(inst)->opsel_hi_2 != 1u) [[unlikely]] return emit_error.emit() << "V_PK_MUL_F64 has an invalid packed F64 element layout"; - if (reinterpret_cast(inst)->src2 != 128u) [[unlikely]] + if (reinterpret_cast(inst)->src2 != 128u || + (reinterpret_cast(inst)->neg & 4u) != 0u || + (reinterpret_cast(inst)->neg_hi & 4u) != 0u) [[unlikely]] return emit_error.emit() << "V_PK_MUL_F64 has an invalid unused src2 encoding"; if (reinterpret_cast(inst)->vdst > 252u) [[unlikely]] return emit_error.emit() @@ -4370,7 +4372,9 @@ DecodeResult decodeVPkAddF64Vop3p(const MachineInst *opcode, const DecodeErrorEm reinterpret_cast(inst)->opsel_hi != 3u || reinterpret_cast(inst)->opsel_hi_2 != 1u) [[unlikely]] return emit_error.emit() << "V_PK_ADD_F64 has an invalid packed F64 element layout"; - if (reinterpret_cast(inst)->src2 != 128u) [[unlikely]] + if (reinterpret_cast(inst)->src2 != 128u || + (reinterpret_cast(inst)->neg & 4u) != 0u || + (reinterpret_cast(inst)->neg_hi & 4u) != 0u) [[unlikely]] return emit_error.emit() << "V_PK_ADD_F64 has an invalid unused src2 encoding"; if (reinterpret_cast(inst)->vdst > 252u) [[unlikely]] return emit_error.emit() @@ -4717,7 +4721,9 @@ DecodeResult decodeVPkMaxNumF64Vop3p(const MachineInst *opcode, reinterpret_cast(inst)->opsel_hi != 3u || reinterpret_cast(inst)->opsel_hi_2 != 1u) [[unlikely]] return emit_error.emit() << "V_PK_MAX_NUM_F64 has an invalid packed F64 element layout"; - if (reinterpret_cast(inst)->src2 != 128u) [[unlikely]] + if (reinterpret_cast(inst)->src2 != 128u || + (reinterpret_cast(inst)->neg & 4u) != 0u || + (reinterpret_cast(inst)->neg_hi & 4u) != 0u) [[unlikely]] return emit_error.emit() << "V_PK_MAX_NUM_F64 has an invalid unused src2 encoding"; if (reinterpret_cast(inst)->vdst > 252u) [[unlikely]] return emit_error.emit() @@ -4830,7 +4836,9 @@ DecodeResult decodeVPkMinNumF64Vop3p(const MachineInst *opcode, reinterpret_cast(inst)->opsel_hi != 3u || reinterpret_cast(inst)->opsel_hi_2 != 1u) [[unlikely]] return emit_error.emit() << "V_PK_MIN_NUM_F64 has an invalid packed F64 element layout"; - if (reinterpret_cast(inst)->src2 != 128u) [[unlikely]] + if (reinterpret_cast(inst)->src2 != 128u || + (reinterpret_cast(inst)->neg & 4u) != 0u || + (reinterpret_cast(inst)->neg_hi & 4u) != 0u) [[unlikely]] return emit_error.emit() << "V_PK_MIN_NUM_F64 has an invalid unused src2 encoding"; if (reinterpret_cast(inst)->vdst > 252u) [[unlikely]] return emit_error.emit() From bea815057755761e01ea4023c6f5409b6defd06f Mon Sep 17 00:00:00 2001 From: hanhanW Date: Tue, 15 Sep 2026 11:55:45 -0700 Subject: [PATCH 6/8] [rocjitsu] Allow SGPR104_128 sources in packed FP64 instructions. LLVM defines each packed FP64 source as VSrc_v2f64. Its VS_128 register class includes SGPR104_128, so rejecting selector 104 makes valid llvm-mc encodings fail to decode. Permit selector 104 in both source positions for V_PK_ADD_F64, V_PK_MUL_F64, V_PK_MAX_NUM_F64, and V_PK_MIN_NUM_F64, then regenerate the CDNA5 decoder. Add fresh-generation assertions for the special selector and unused modifier bits. Exercise each operation with SGPR104_128 in both source positions and verify that the VCC tuple components are not consumed. Tests: Gfx1251PackedF64ExecutionTest.*; related generator profile tests --- .../lib/python/amdisa/codegen/_generator.py | 15 +--- .../tests/test_generator_profile_gates.py | 85 +++++++++++-------- .../isa/arch/amdgpu/generated/cdna5/vop3p.cpp | 32 +++++-- .../rocjitsu/tests/cdna5_execution_test.cpp | 71 +++++++++++++++- 4 files changed, 143 insertions(+), 60 deletions(-) diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py index 39402165a60e..5068fedac7e8 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py @@ -10416,10 +10416,6 @@ def gen_insts(self) -> None: if inst_sem.semantic_class == 'pk_binop_f64' else 'packed U64' ) - allows_sgpr104_128 = inst_sem.semantic_class in { - 'pk_binop_u64', - 'pk_lshl_add_u64', - } factory_validation_parts.append( f'if ({raw_inst}->opsel != 0u || ' f'{raw_inst}->opsel_hi != 3u || ' @@ -10461,9 +10457,9 @@ def gen_insts(self) -> None: max_sgpr = 106 - register_count max_vgpr = 512 - register_count invalid_sgpr_span = f'{raw_value} > {max_sgpr}u' - if opnd.size == 128 and allows_sgpr104_128: + if opnd.size == 128: # SGPR104_128 is a named VS_128 member: - # s104:s105 provide the scalar U64 value, + # s104:s105 provide the scalar 64-bit value, # while vcc_lo:vcc_hi complete the tuple. invalid_sgpr_span = ( f'({invalid_sgpr_span} && {raw_value} != 104u)' @@ -10493,14 +10489,9 @@ def gen_insts(self) -> None: ) if opnd.operand_type == 'OPR_SRC': if opnd.size == 128: - sgpr104_selector = ( - f'{raw_value} == 104u || ' - if allows_sgpr104_128 - else '' - ) valid_width_specific = ( f'({raw_value} <= 100u && ({raw_value} % 4u) == 0u) || ' - f'{sgpr104_selector}' + f'{raw_value} == 104u || ' f'({raw_value} >= 108u && {raw_value} <= 120u && ' f'(({raw_value} - 108u) % 4u) == 0u) || ' f'{raw_value} == 124u || ' diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py index f4d3f532017c..b6d5913d13a6 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py @@ -5263,44 +5263,57 @@ def test_gfx1251_packed_u64_decode_rejects_undefined_layouts_and_register_tuples def test_gfx1251_packed_f64_decode_rejects_undefined_layouts_and_register_tuples( - gfx1250_generated_root: Path, + gfx1250_generated_root: Path, tmp_path: Path ): - source = (gfx1250_generated_root / 'vop3p.cpp').read_text() + args = SimpleNamespace( + isafiles=[f'cdna5:{_mrisa_dir() / "amdgpu_isa_cdna5.xml"}'], + isa_additions=[f'cdna5:{_mrisa_dir() / "amdgpu_isa_cdna5_gfx1251_delta.xml"}'], + isa_variants=[f'cdna5:{_mrisa_dir() / "amdgpu_isa_cdna5_variants.json"}'], + gen_isas=True, + gen_dbt=False, + isa_output=str(tmp_path), + dbt_output=None, + ) + _run(args) - for class_name in ( - 'VPkAddF64Vop3p', - 'VPkMulF64Vop3p', - 'VPkMaxNumF64Vop3p', - 'VPkMinNumF64Vop3p', - ): - body = _generated_decode_body(source, class_name) - assert 'has an invalid packed F64 element layout' in body - assert 'has an invalid unused src2 encoding' in body - assert 'vdst register tuple that exceeds the selector range' in body - assert 'src0 register tuple that exceeds the selector range' in body - assert 'src1 register tuple that exceeds the selector range' in body - assert 'invalid vdst register tuple alignment' in body - assert 'invalid src0 register tuple alignment' in body - assert 'invalid src1 register tuple alignment' in body - assert 'invalid src0 packed F64 source selector' in body - assert 'invalid src1 packed F64 source selector' in body - for operand_name in ('src0', 'src1'): - assert f'{operand_name} != 104u' not in body - assert f'{operand_name} == 104u' not in body - assert f'{operand_name} <= 100u' in body - assert f'{operand_name} >= 108u' in body - assert f'{operand_name} <= 120u' in body - assert f'{operand_name} == 124u' in body - assert f'{operand_name} >= 128u' in body - assert f'{operand_name} <= 208u' in body - assert f'{operand_name} >= 240u' in body - assert f'{operand_name} <= 248u' in body - assert f'{operand_name} == 255u' in body - assert f'{operand_name} >= 256u' in body - assert f'{operand_name} <= 508u' in body - for invalid_selector in (230, 231, 235, 236, 253): - assert f'{operand_name} == {invalid_selector}u' not in body - assert 'does not support source modifiers or clamp' not in body + for generated_root in (gfx1250_generated_root, tmp_path / 'cdna5'): + source = (generated_root / 'vop3p.cpp').read_text() + for class_name in ( + 'VPkAddF64Vop3p', + 'VPkMulF64Vop3p', + 'VPkMaxNumF64Vop3p', + 'VPkMinNumF64Vop3p', + ): + body = _generated_decode_body(source, class_name) + assert 'has an invalid packed F64 element layout' in body + assert 'has an invalid unused src2 encoding' in body + assert '->neg & 4u) != 0u' in body + assert '->neg_hi & 4u) != 0u' in body + assert 'vdst register tuple that exceeds the selector range' in body + assert 'src0 register tuple that exceeds the selector range' in body + assert 'src1 register tuple that exceeds the selector range' in body + assert 'invalid vdst register tuple alignment' in body + assert 'invalid src0 register tuple alignment' in body + assert 'invalid src1 register tuple alignment' in body + assert 'invalid src0 packed F64 source selector' in body + assert 'invalid src1 packed F64 source selector' in body + for operand_name in ('src0', 'src1'): + assert f'{operand_name} != 104u' in body + assert f'{operand_name} == 104u' in body + assert f'{operand_name} <= 100u' in body + assert f'{operand_name} >= 108u' in body + assert f'{operand_name} <= 120u' in body + assert f'{operand_name} == 124u' in body + assert f'{operand_name} >= 128u' in body + assert f'{operand_name} <= 208u' in body + assert f'{operand_name} >= 240u' in body + assert f'{operand_name} <= 248u' in body + assert f'{operand_name} == 255u' in body + assert f'{operand_name} >= 256u' in body + assert f'{operand_name} <= 508u' in body + for invalid_selector in (230, 231, 235, 236, 253): + assert f'{operand_name} == {invalid_selector}u' not in body + assert 'does not support source modifiers or clamp' not in body def test_gfx1251_packed_f64_literals_use_f64_high_bits_widening( diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.cpp index cd710e396e49..80b4c3ace7f2 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.cpp @@ -4268,7 +4268,8 @@ DecodeResult decodeVPkMulF64Vop3p(const MachineInst *opcode, const DecodeErrorEm if ((reinterpret_cast(inst)->vdst & 1u) != 0u) [[unlikely]] return emit_error.emit() << "V_PK_MUL_F64 has an invalid vdst register tuple alignment"; if ((reinterpret_cast(inst)->src0 <= 105u && - reinterpret_cast(inst)->src0 > 102u) || + (reinterpret_cast(inst)->src0 > 102u && + reinterpret_cast(inst)->src0 != 104u)) || (reinterpret_cast(inst)->src0 >= 256u && reinterpret_cast(inst)->src0 > 508u)) [[unlikely]] return emit_error.emit() @@ -4283,6 +4284,7 @@ DecodeResult decodeVPkMulF64Vop3p(const MachineInst *opcode, const DecodeErrorEm return emit_error.emit() << "V_PK_MUL_F64 has an invalid src0 register tuple alignment"; if (!((reinterpret_cast(inst)->src0 <= 100u && (reinterpret_cast(inst)->src0 % 4u) == 0u) || + reinterpret_cast(inst)->src0 == 104u || (reinterpret_cast(inst)->src0 >= 108u && reinterpret_cast(inst)->src0 <= 120u && ((reinterpret_cast(inst)->src0 - 108u) % 4u) == 0u) || @@ -4297,7 +4299,8 @@ DecodeResult decodeVPkMulF64Vop3p(const MachineInst *opcode, const DecodeErrorEm (reinterpret_cast(inst)->src0 & 1u) == 0u))) [[unlikely]] return emit_error.emit() << "V_PK_MUL_F64 has an invalid src0 packed F64 source selector"; if ((reinterpret_cast(inst)->src1 <= 105u && - reinterpret_cast(inst)->src1 > 102u) || + (reinterpret_cast(inst)->src1 > 102u && + reinterpret_cast(inst)->src1 != 104u)) || (reinterpret_cast(inst)->src1 >= 256u && reinterpret_cast(inst)->src1 > 508u)) [[unlikely]] return emit_error.emit() @@ -4312,6 +4315,7 @@ DecodeResult decodeVPkMulF64Vop3p(const MachineInst *opcode, const DecodeErrorEm return emit_error.emit() << "V_PK_MUL_F64 has an invalid src1 register tuple alignment"; if (!((reinterpret_cast(inst)->src1 <= 100u && (reinterpret_cast(inst)->src1 % 4u) == 0u) || + reinterpret_cast(inst)->src1 == 104u || (reinterpret_cast(inst)->src1 >= 108u && reinterpret_cast(inst)->src1 <= 120u && ((reinterpret_cast(inst)->src1 - 108u) % 4u) == 0u) || @@ -4382,7 +4386,8 @@ DecodeResult decodeVPkAddF64Vop3p(const MachineInst *opcode, const DecodeErrorEm if ((reinterpret_cast(inst)->vdst & 1u) != 0u) [[unlikely]] return emit_error.emit() << "V_PK_ADD_F64 has an invalid vdst register tuple alignment"; if ((reinterpret_cast(inst)->src0 <= 105u && - reinterpret_cast(inst)->src0 > 102u) || + (reinterpret_cast(inst)->src0 > 102u && + reinterpret_cast(inst)->src0 != 104u)) || (reinterpret_cast(inst)->src0 >= 256u && reinterpret_cast(inst)->src0 > 508u)) [[unlikely]] return emit_error.emit() @@ -4397,6 +4402,7 @@ DecodeResult decodeVPkAddF64Vop3p(const MachineInst *opcode, const DecodeErrorEm return emit_error.emit() << "V_PK_ADD_F64 has an invalid src0 register tuple alignment"; if (!((reinterpret_cast(inst)->src0 <= 100u && (reinterpret_cast(inst)->src0 % 4u) == 0u) || + reinterpret_cast(inst)->src0 == 104u || (reinterpret_cast(inst)->src0 >= 108u && reinterpret_cast(inst)->src0 <= 120u && ((reinterpret_cast(inst)->src0 - 108u) % 4u) == 0u) || @@ -4411,7 +4417,8 @@ DecodeResult decodeVPkAddF64Vop3p(const MachineInst *opcode, const DecodeErrorEm (reinterpret_cast(inst)->src0 & 1u) == 0u))) [[unlikely]] return emit_error.emit() << "V_PK_ADD_F64 has an invalid src0 packed F64 source selector"; if ((reinterpret_cast(inst)->src1 <= 105u && - reinterpret_cast(inst)->src1 > 102u) || + (reinterpret_cast(inst)->src1 > 102u && + reinterpret_cast(inst)->src1 != 104u)) || (reinterpret_cast(inst)->src1 >= 256u && reinterpret_cast(inst)->src1 > 508u)) [[unlikely]] return emit_error.emit() @@ -4426,6 +4433,7 @@ DecodeResult decodeVPkAddF64Vop3p(const MachineInst *opcode, const DecodeErrorEm return emit_error.emit() << "V_PK_ADD_F64 has an invalid src1 register tuple alignment"; if (!((reinterpret_cast(inst)->src1 <= 100u && (reinterpret_cast(inst)->src1 % 4u) == 0u) || + reinterpret_cast(inst)->src1 == 104u || (reinterpret_cast(inst)->src1 >= 108u && reinterpret_cast(inst)->src1 <= 120u && ((reinterpret_cast(inst)->src1 - 108u) % 4u) == 0u) || @@ -4731,7 +4739,8 @@ DecodeResult decodeVPkMaxNumF64Vop3p(const MachineInst *opcode, if ((reinterpret_cast(inst)->vdst & 1u) != 0u) [[unlikely]] return emit_error.emit() << "V_PK_MAX_NUM_F64 has an invalid vdst register tuple alignment"; if ((reinterpret_cast(inst)->src0 <= 105u && - reinterpret_cast(inst)->src0 > 102u) || + (reinterpret_cast(inst)->src0 > 102u && + reinterpret_cast(inst)->src0 != 104u)) || (reinterpret_cast(inst)->src0 >= 256u && reinterpret_cast(inst)->src0 > 508u)) [[unlikely]] return emit_error.emit() @@ -4746,6 +4755,7 @@ DecodeResult decodeVPkMaxNumF64Vop3p(const MachineInst *opcode, return emit_error.emit() << "V_PK_MAX_NUM_F64 has an invalid src0 register tuple alignment"; if (!((reinterpret_cast(inst)->src0 <= 100u && (reinterpret_cast(inst)->src0 % 4u) == 0u) || + reinterpret_cast(inst)->src0 == 104u || (reinterpret_cast(inst)->src0 >= 108u && reinterpret_cast(inst)->src0 <= 120u && ((reinterpret_cast(inst)->src0 - 108u) % 4u) == 0u) || @@ -4760,7 +4770,8 @@ DecodeResult decodeVPkMaxNumF64Vop3p(const MachineInst *opcode, (reinterpret_cast(inst)->src0 & 1u) == 0u))) [[unlikely]] return emit_error.emit() << "V_PK_MAX_NUM_F64 has an invalid src0 packed F64 source selector"; if ((reinterpret_cast(inst)->src1 <= 105u && - reinterpret_cast(inst)->src1 > 102u) || + (reinterpret_cast(inst)->src1 > 102u && + reinterpret_cast(inst)->src1 != 104u)) || (reinterpret_cast(inst)->src1 >= 256u && reinterpret_cast(inst)->src1 > 508u)) [[unlikely]] return emit_error.emit() @@ -4775,6 +4786,7 @@ DecodeResult decodeVPkMaxNumF64Vop3p(const MachineInst *opcode, return emit_error.emit() << "V_PK_MAX_NUM_F64 has an invalid src1 register tuple alignment"; if (!((reinterpret_cast(inst)->src1 <= 100u && (reinterpret_cast(inst)->src1 % 4u) == 0u) || + reinterpret_cast(inst)->src1 == 104u || (reinterpret_cast(inst)->src1 >= 108u && reinterpret_cast(inst)->src1 <= 120u && ((reinterpret_cast(inst)->src1 - 108u) % 4u) == 0u) || @@ -4846,7 +4858,8 @@ DecodeResult decodeVPkMinNumF64Vop3p(const MachineInst *opcode, if ((reinterpret_cast(inst)->vdst & 1u) != 0u) [[unlikely]] return emit_error.emit() << "V_PK_MIN_NUM_F64 has an invalid vdst register tuple alignment"; if ((reinterpret_cast(inst)->src0 <= 105u && - reinterpret_cast(inst)->src0 > 102u) || + (reinterpret_cast(inst)->src0 > 102u && + reinterpret_cast(inst)->src0 != 104u)) || (reinterpret_cast(inst)->src0 >= 256u && reinterpret_cast(inst)->src0 > 508u)) [[unlikely]] return emit_error.emit() @@ -4861,6 +4874,7 @@ DecodeResult decodeVPkMinNumF64Vop3p(const MachineInst *opcode, return emit_error.emit() << "V_PK_MIN_NUM_F64 has an invalid src0 register tuple alignment"; if (!((reinterpret_cast(inst)->src0 <= 100u && (reinterpret_cast(inst)->src0 % 4u) == 0u) || + reinterpret_cast(inst)->src0 == 104u || (reinterpret_cast(inst)->src0 >= 108u && reinterpret_cast(inst)->src0 <= 120u && ((reinterpret_cast(inst)->src0 - 108u) % 4u) == 0u) || @@ -4875,7 +4889,8 @@ DecodeResult decodeVPkMinNumF64Vop3p(const MachineInst *opcode, (reinterpret_cast(inst)->src0 & 1u) == 0u))) [[unlikely]] return emit_error.emit() << "V_PK_MIN_NUM_F64 has an invalid src0 packed F64 source selector"; if ((reinterpret_cast(inst)->src1 <= 105u && - reinterpret_cast(inst)->src1 > 102u) || + (reinterpret_cast(inst)->src1 > 102u && + reinterpret_cast(inst)->src1 != 104u)) || (reinterpret_cast(inst)->src1 >= 256u && reinterpret_cast(inst)->src1 > 508u)) [[unlikely]] return emit_error.emit() @@ -4890,6 +4905,7 @@ DecodeResult decodeVPkMinNumF64Vop3p(const MachineInst *opcode, return emit_error.emit() << "V_PK_MIN_NUM_F64 has an invalid src1 register tuple alignment"; if (!((reinterpret_cast(inst)->src1 <= 100u && (reinterpret_cast(inst)->src1 % 4u) == 0u) || + reinterpret_cast(inst)->src1 == 104u || (reinterpret_cast(inst)->src1 >= 108u && reinterpret_cast(inst)->src1 <= 120u && ((reinterpret_cast(inst)->src1 - 108u) % 4u) == 0u) || diff --git a/emulation/rocjitsu/tests/cdna5_execution_test.cpp b/emulation/rocjitsu/tests/cdna5_execution_test.cpp index 290bc2d70cf9..7048ee2b0bbd 100644 --- a/emulation/rocjitsu/tests/cdna5_execution_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_execution_test.cpp @@ -2488,6 +2488,57 @@ TEST(Gfx1251PackedF64ExecutionTest, MinMaxNumberHandleNanInfinityAndSignedZero) } } +TEST(Gfx1251PackedF64ExecutionTest, ExecutesSgpr104CompositeInBothSourcesForEveryOperation) { + constexpr auto bits = [](double value) { return std::bit_cast(value); }; + struct Operation { + std::string_view name; + uint32_t first_word; + PackedU64Pair expected; + }; + constexpr std::array kOperations{ + Operation{"add", 0xCC4B4004u, {bits(5.0), bits(6.0)}}, + Operation{"mul", 0xCC3C4004u, {bits(6.0), bits(8.0)}}, + Operation{"max-num", 0xCC4E4004u, {bits(3.0), bits(4.0)}}, + Operation{"min-num", 0xCC4F4004u, {bits(2.0), bits(2.0)}}, + }; + // Exact encodings produced by public LLVM MC for the SGPR104_128 composite + // ([s104,s105,vcc_lo,vcc_hi]) in both VSrc_v2f64 positions. + constexpr uint32_t kSgpr104Src0 = 0x1A021068u; + constexpr uint32_t kSgpr104Src1 = 0x1A00D108u; + + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + + constexpr uint64_t kVccPoison = 0xdeadbeefcafef00dULL; + write_wave_sgpr(*cu, *wf, 104, static_cast(bits(2.0))); + write_wave_sgpr(*cu, *wf, 105, static_cast(bits(2.0) >> 32)); + wf->set_vcc_raw(kVccPoison); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {bits(3.0), bits(4.0)}); + + for (const auto &operation : kOperations) { + for (const bool use_src1 : {false, true}) { + SCOPED_TRACE(operation.name); + SCOPED_TRACE(use_src1 ? "src1" : "src0"); + const std::array words{ + operation.first_word, + use_src1 ? kSgpr104Src1 : kSgpr104Src0, + }; + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), operation.expected); + EXPECT_EQ(wf->vcc(), kVccPoison); + } + } +} + TEST(Gfx1251PackedF64ExecutionTest, RejectsUndefinedLayoutsAndOutOfRangeRegisterTuples) { auto decoder = make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); @@ -2514,8 +2565,20 @@ TEST(Gfx1251PackedF64ExecutionTest, RejectsUndefinedLayoutsAndOutOfRangeRegister .vdst = 253, .opsel_hi_2 = 1, .src0 = 264, .src1 = 268, .src2 = 128, .opsel_hi = 3}, cdna5::Vop3pBuilderFields{ .vdst = 4, .opsel_hi_2 = 1, .src0 = 509, .src1 = 268, .src2 = 128, .opsel_hi = 3}, - cdna5::Vop3pBuilderFields{ - .vdst = 4, .opsel_hi_2 = 1, .src0 = 264, .src1 = 104, .src2 = 128, .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{.vdst = 4, + .neg_hi = 4, + .opsel_hi_2 = 1, + .src0 = 264, + .src1 = 268, + .src2 = 128, + .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{.vdst = 4, + .opsel_hi_2 = 1, + .src0 = 264, + .src1 = 268, + .src2 = 128, + .opsel_hi = 3, + .neg = 4}, cdna5::Vop3pBuilderFields{ .vdst = 3, .opsel_hi_2 = 1, .src0 = 264, .src1 = 268, .src2 = 128, .opsel_hi = 3}, cdna5::Vop3pBuilderFields{ @@ -2554,8 +2617,8 @@ TEST(Gfx1251PackedF64ExecutionTest, RejectsUndefinedLayoutsAndOutOfRangeRegister } } - constexpr std::array kValidV2F64Selectors{ - 124, 128, 208, 240, 248, 255, + constexpr std::array kValidV2F64Selectors{ + 104, 124, 128, 208, 240, 248, 255, }; for (const uint16_t opcode : kOpcodes) { SCOPED_TRACE(opcode); From c306de0e98ca427d967ed1bc4c14f9af9a84a97b Mon Sep 17 00:00:00 2001 From: hanhanW Date: Wed, 16 Sep 2026 11:01:59 -0700 Subject: [PATCH 7/8] [rocjitsu] Define SIMM32 widening by operand signature. Packed F64 sources are 128-bit operands whose 32-bit literal supplies a broadcast F64 element. The generator previously repeated FMT_NUM_PK2_F64 in both the widening table and a size predicate, while the surrounding names and diagnostic still described only 64-bit operands. Key widening policies by operand width and data format so the packed-F64 table entry is the single source of truth. Generalize the helper name and diagnostic, and add focused coverage showing that 128-bit packed F64 uses F64 high-bit widening while packed U64 retains the default immediate semantics. Tests: test_literal_codegen.py; relevant generator profile tests; generate-amdisa.sh --- .../lib/python/amdisa/codegen/_generator.py | 86 ++++++++++--------- .../amdisa/tests/test_literal_codegen.py | 30 ++++++- 2 files changed, 72 insertions(+), 44 deletions(-) diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py index 5068fedac7e8..f780829e8e15 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py @@ -109,38 +109,39 @@ class _Literal32Widening(Enum): F64_HIGH_BITS = 'F64HighBits' -# Keep every shipped 64-bit source format explicit. PK_F32 replicates its literal -# into both DWORDs, while I64 and F64 reinterpret the extension word by type. -_SIMM32_64BIT_WIDENING = { - 'FMT_NUM_B32': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_B64': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_BF16': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_BF8': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_F16': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_F32': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_FP8': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_I8': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_M64': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_PK16_U4': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_PK2_B32': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_PK2_F32': _Literal32Widening.REPLICATE_32, - 'FMT_NUM_PK2_F64': _Literal32Widening.F64_HIGH_BITS, - 'FMT_NUM_PK2_U32': _Literal32Widening.REPLICATE_32, - 'FMT_NUM_PK4_BF16': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_PK4_F16': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_PK8_BF8': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_PK8_FP8': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_PK8_I8': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_U64': _Literal32Widening.ZERO_EXTEND, - 'FMT_WMMA_AB_16X16_BF8': _Literal32Widening.ZERO_EXTEND, - 'FMT_WMMA_AB_16X16_FP8': _Literal32Widening.ZERO_EXTEND, - 'FMT_WMMA_AB_16X16_IU8': _Literal32Widening.ZERO_EXTEND, - 'FMT_WMMA_AB_16X32_IU4': _Literal32Widening.ZERO_EXTEND, - 'FMT_WMMA_AB_16X4_F32': _Literal32Widening.ZERO_EXTEND, - 'FMT_WMMA_AB_IU4': _Literal32Widening.ZERO_EXTEND, - 'FMT_WMMA_INDEX_SET2': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_I64': _Literal32Widening.SIGN_EXTEND, - 'FMT_NUM_F64': _Literal32Widening.F64_HIGH_BITS, +# Keep every shipped wide-source signature explicit. PK_F32 replicates its +# literal into both DWORDs, while I64 and F64 reinterpret the extension word by +# type. Packed F64 is a 128-bit operand containing two broadcast 64-bit values. +_SIMM32_LITERAL_WIDENING = { + (64, 'FMT_NUM_B32'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_B64'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_BF16'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_BF8'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_F16'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_F32'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_FP8'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_I8'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_M64'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_PK16_U4'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_PK2_B32'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_PK2_F32'): _Literal32Widening.REPLICATE_32, + (64, 'FMT_NUM_PK2_U32'): _Literal32Widening.REPLICATE_32, + (64, 'FMT_NUM_PK4_BF16'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_PK4_F16'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_PK8_BF8'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_PK8_FP8'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_PK8_I8'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_U64'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_WMMA_AB_16X16_BF8'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_WMMA_AB_16X16_FP8'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_WMMA_AB_16X16_IU8'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_WMMA_AB_16X32_IU4'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_WMMA_AB_16X4_F32'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_WMMA_AB_IU4'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_WMMA_INDEX_SET2'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_I64'): _Literal32Widening.SIGN_EXTEND, + (64, 'FMT_NUM_F64'): _Literal32Widening.F64_HIGH_BITS, + (128, 'FMT_NUM_PK2_F64'): _Literal32Widening.F64_HIGH_BITS, } _LITERAL_ENCODING_OPERANDS = { @@ -1210,7 +1211,7 @@ def _literal_operand_from_expr_stmt( ) if operand_type == 'OPR_SIMM16' or opnd.size == 16: literal_expr = f'({literal_expr} & 0xFFFFu)' - widening = CodeGenerator._literal_operand_64bit_widening( + widening = CodeGenerator._literal_operand_simm32_widening( opnd, operand_type, arch_name, inst_name, enc_name ) if widening is not None: @@ -1225,26 +1226,29 @@ def _literal_operand_from_expr_stmt( ) @staticmethod - def _literal_operand_64bit_widening( + def _literal_operand_simm32_widening( opnd: Operand, operand_type: str, arch_name: str, inst_name: str, enc_name: str, ) -> _Literal32Widening | None: - is_64bit_value = opnd.size == 64 or ( - opnd.size == 128 and opnd.data_format_name == 'FMT_NUM_PK2_F64' - ) - if operand_type != 'OPR_SIMM32' or not opnd.is_input or not is_64bit_value: + if operand_type != 'OPR_SIMM32' or not opnd.is_input: return None + signature = (opnd.size, opnd.data_format_name) try: - return _SIMM32_64BIT_WIDENING[opnd.data_format_name] + return _SIMM32_LITERAL_WIDENING[signature] except KeyError as exc: + # Every 64-bit SIMM32 input requires an explicit policy. Wider + # packed inputs retain the default immediate semantics unless + # listed above. + if opnd.size != 64: + return None data_format = opnd.data_format_name or '' raise ValueError( f'architecture {arch_name!r}, instruction {inst_name!r}, encoding ' - f'{enc_name!r}: 64-bit SIMM32 input operand {opnd.name!r} has ' - f'unsupported data format {data_format!r}' + f'{enc_name!r}: SIMM32 input operand {opnd.name!r} has no widening ' + f'policy for size {opnd.size} and data format {data_format!r}' ) from exc @staticmethod diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_literal_codegen.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_literal_codegen.py index 793ab4e503c1..36d87b067a64 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_literal_codegen.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_literal_codegen.py @@ -7,7 +7,7 @@ import pytest -from amdisa.codegen._generator import CodeGenerator +from amdisa.codegen._generator import CodeGenerator, _Literal32Widening from amdisa.codegen.execute.simd_codegen import ( SIMD_VOP2_TERNARY, simd_ternary_literal_operand_name, @@ -272,12 +272,35 @@ def test_f64_simm32_literal_operand_uses_extension_word_as_double_high_bits(): ) == stmt +def test_128bit_packed_f64_widens_simm32_but_packed_u64_does_not(): + packed_f64 = _operand( + 'src0', 'OPR_SRC', size=128, data_format_name='FMT_NUM_PK2_F64' + ) + packed_u64 = _operand( + 'src0', 'OPR_SRC', size=128, data_format_name='FMT_NUM_PK2_U64' + ) + + assert ( + CodeGenerator._literal_operand_simm32_widening( + packed_f64, 'OPR_SIMM32', 'cdna5', 'V_PK_ADD_F64', 'ENC_VOP3P' + ) + is _Literal32Widening.F64_HIGH_BITS + ) + assert ( + CodeGenerator._literal_operand_simm32_widening( + packed_u64, 'OPR_SIMM32', 'cdna5', 'V_PK_ADD_U64', 'ENC_VOP3P' + ) + is None + ) + + def test_64bit_simm32_literal_operand_requires_data_format(): with pytest.raises( ValueError, match=( "architecture 'rdna4', instruction 'V_FMAC_F64', encoding 'ENC_VOP2': " - "64-bit SIMM32 input operand 'src0' has unsupported data format ''" + "SIMM32 input operand 'src0' has no widening policy for size 64 and " + "data format ''" ), ): CodeGenerator._literal_operand_fixup_stmt( @@ -400,7 +423,8 @@ def test_64bit_simm32_literal_operand_rejects_unrecognized_data_format(): ValueError, match=( "architecture 'cdna4', instruction 'S_FAKE_I64', encoding 'ENC_SOP2': " - "64-bit SIMM32 input operand 'ssrc0' has unsupported data format 'FMT_NUM_X64'" + "SIMM32 input operand 'ssrc0' has no widening policy for size 64 and " + "data format 'FMT_NUM_X64'" ), ): CodeGenerator._literal_operand_fixup_stmt( From afde462cb7ae55830f49cfe224c70893c7dca577 Mon Sep 17 00:00:00 2001 From: hanhanW Date: Wed, 16 Sep 2026 11:42:47 -0700 Subject: [PATCH 8/8] [rocjitsu] Avoid a GCC false positive in the DBI argument test. GCC 13 at -O2 diagnoses the single-element std::vector initializer-list assignment in DbiArgSimBase::SetUp as an out-of-bounds inlined memmove. Because rocJitsu treats warnings as errors, this prevents the ubsan-gcc CI lane from building after the DBI argument tests landed on develop. Append the sentinel to the newly constructed empty vector instead. This preserves the one-element probe argument list while avoiding the optimizer warning. Tests: GCC 13/UBSan full build; DBI argument simulator tests; packed-F64 execution tests --- emulation/rocjitsu/tests/dbi/dbi_arg_sim_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emulation/rocjitsu/tests/dbi/dbi_arg_sim_test.cpp b/emulation/rocjitsu/tests/dbi/dbi_arg_sim_test.cpp index 71360e12190c..3d5a0c0b4990 100644 --- a/emulation/rocjitsu/tests/dbi/dbi_arg_sim_test.cpp +++ b/emulation/rocjitsu/tests/dbi/dbi_arg_sim_test.cpp @@ -109,7 +109,7 @@ class DbiArgSimBase : public ::testing::Test { pt.anchor_offset = 4; // v_mov_b32 v1, v0 -> reads v0 (v0 live at the anchor). pt.probe_obj = &probe_obj; pt.probe_symbol = "rj_test_arg_probe"; - pt.probe_args = {kArgSentinel}; + pt.probe_args.push_back(kArgSentinel); instr.add_point(pt); auto result = instr.patch_with_debug_summaries();