diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py index 94982d63969c..f780829e8e15 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py @@ -109,37 +109,39 @@ class _Literal32Widening(Enum): F64_HIGH_BITS = 'F64HighBits' -# Keep every shipped 64-bit source format explicit. PK_F32 replicates its literal -# into both DWORDs, while I64 and F64 reinterpret the extension word by type. -_SIMM32_64BIT_WIDENING = { - 'FMT_NUM_B32': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_B64': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_BF16': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_BF8': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_F16': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_F32': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_FP8': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_I8': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_M64': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_PK16_U4': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_PK2_B32': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_PK2_F32': _Literal32Widening.REPLICATE_32, - 'FMT_NUM_PK2_U32': _Literal32Widening.REPLICATE_32, - 'FMT_NUM_PK4_BF16': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_PK4_F16': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_PK8_BF8': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_PK8_FP8': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_PK8_I8': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_U64': _Literal32Widening.ZERO_EXTEND, - 'FMT_WMMA_AB_16X16_BF8': _Literal32Widening.ZERO_EXTEND, - 'FMT_WMMA_AB_16X16_FP8': _Literal32Widening.ZERO_EXTEND, - 'FMT_WMMA_AB_16X16_IU8': _Literal32Widening.ZERO_EXTEND, - 'FMT_WMMA_AB_16X32_IU4': _Literal32Widening.ZERO_EXTEND, - 'FMT_WMMA_AB_16X4_F32': _Literal32Widening.ZERO_EXTEND, - 'FMT_WMMA_AB_IU4': _Literal32Widening.ZERO_EXTEND, - 'FMT_WMMA_INDEX_SET2': _Literal32Widening.ZERO_EXTEND, - 'FMT_NUM_I64': _Literal32Widening.SIGN_EXTEND, - 'FMT_NUM_F64': _Literal32Widening.F64_HIGH_BITS, +# Keep every shipped wide-source signature explicit. PK_F32 replicates its +# literal into both DWORDs, while I64 and F64 reinterpret the extension word by +# type. Packed F64 is a 128-bit operand containing two broadcast 64-bit values. +_SIMM32_LITERAL_WIDENING = { + (64, 'FMT_NUM_B32'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_B64'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_BF16'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_BF8'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_F16'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_F32'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_FP8'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_I8'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_M64'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_PK16_U4'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_PK2_B32'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_PK2_F32'): _Literal32Widening.REPLICATE_32, + (64, 'FMT_NUM_PK2_U32'): _Literal32Widening.REPLICATE_32, + (64, 'FMT_NUM_PK4_BF16'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_PK4_F16'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_PK8_BF8'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_PK8_FP8'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_PK8_I8'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_U64'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_WMMA_AB_16X16_BF8'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_WMMA_AB_16X16_FP8'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_WMMA_AB_16X16_IU8'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_WMMA_AB_16X32_IU4'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_WMMA_AB_16X4_F32'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_WMMA_AB_IU4'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_WMMA_INDEX_SET2'): _Literal32Widening.ZERO_EXTEND, + (64, 'FMT_NUM_I64'): _Literal32Widening.SIGN_EXTEND, + (64, 'FMT_NUM_F64'): _Literal32Widening.F64_HIGH_BITS, + (128, 'FMT_NUM_PK2_F64'): _Literal32Widening.F64_HIGH_BITS, } _LITERAL_ENCODING_OPERANDS = { @@ -1209,7 +1211,7 @@ def _literal_operand_from_expr_stmt( ) if operand_type == 'OPR_SIMM16' or opnd.size == 16: literal_expr = f'({literal_expr} & 0xFFFFu)' - widening = CodeGenerator._literal_operand_64bit_widening( + widening = CodeGenerator._literal_operand_simm32_widening( opnd, operand_type, arch_name, inst_name, enc_name ) if widening is not None: @@ -1224,23 +1226,29 @@ def _literal_operand_from_expr_stmt( ) @staticmethod - def _literal_operand_64bit_widening( + def _literal_operand_simm32_widening( opnd: Operand, operand_type: str, arch_name: str, inst_name: str, enc_name: str, ) -> _Literal32Widening | None: - if operand_type != 'OPR_SIMM32' or not opnd.is_input or opnd.size != 64: + if operand_type != 'OPR_SIMM32' or not opnd.is_input: return None + signature = (opnd.size, opnd.data_format_name) try: - return _SIMM32_64BIT_WIDENING[opnd.data_format_name] + return _SIMM32_LITERAL_WIDENING[signature] except KeyError as exc: + # Every 64-bit SIMM32 input requires an explicit policy. Wider + # packed inputs retain the default immediate semantics unless + # listed above. + if opnd.size != 64: + return None data_format = opnd.data_format_name or '' raise ValueError( f'architecture {arch_name!r}, instruction {inst_name!r}, encoding ' - f'{enc_name!r}: 64-bit SIMM32 input operand {opnd.name!r} has ' - f'unsupported data format {data_format!r}' + f'{enc_name!r}: SIMM32 input operand {opnd.name!r} has no widening ' + f'policy for size {opnd.size} and data format {data_format!r}' ) from exc @staticmethod @@ -10383,12 +10391,12 @@ def gen_insts(self) -> None: f'<< "{inst.name} requires its GLOBAL segment";' ) - # LLVM's public gfx1251 profiles define the packed U64 - # operations without op_sel. The profile and corresponding - # MC vectors are permanently linked here: - # https://github.com/llvm/llvm-project/blob/3bcd9a803184e2d3657b9d5cc2a1773e9ce0f116/llvm/lib/Target/AMDGPU/VOP3PInstructions.td#L147-L162 - # https://github.com/llvm/llvm-project/blob/3bcd9a803184e2d3657b9d5cc2a1773e9ce0f116/llvm/test/MC/AMDGPU/gfx1251_asm_vop3p.s#L257-L403 - # https://github.com/llvm/llvm-project/blob/551d5172dd3902efbce5f4720b75bfc4e6441dc8/llvm/lib/Target/AMDGPU/SIRegisterInfo.td#L887-L917 + # LLVM's public gfx1251 profiles define the packed U64 and + # F64 binary operations without op_sel. The profile and + # corresponding MC vectors are permanently linked here: + # https://github.com/llvm/llvm-project/blob/3bcd9a803184e2d3657b9d5cc2a1773e9ce0f116/llvm/lib/Target/AMDGPU/VOP3PInstructions.td#L130-L162 + # https://github.com/llvm/llvm-project/blob/3bcd9a803184e2d3657b9d5cc2a1773e9ce0f116/llvm/test/MC/AMDGPU/gfx1251_asm_vop3p.s#L1-L403 + # https://github.com/llvm/llvm-project/blob/551d5172dd3902efbce5f4720b75bfc4e6441dc8/llvm/lib/Target/AMDGPU/SIRegisterInfo.td#L875-L920 # https://github.com/llvm/llvm-project/blob/551d5172dd3902efbce5f4720b75bfc4e6441dc8/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp#L2133-L2143 # These references establish the operand profile and # encoding, not execution ordering for combined modifiers. @@ -10400,20 +10408,26 @@ def gen_insts(self) -> None: # tuple's first selector and its width-independent source # selector set. if inst_sem is not None and inst_sem.semantic_class in { + 'pk_binop_f64', 'pk_binop_u64', 'pk_lshl_add_u64', }: raw_inst = ( f'reinterpret_cast(inst)' ) + layout_name = ( + 'packed F64' + if inst_sem.semantic_class == 'pk_binop_f64' + else 'packed U64' + ) factory_validation_parts.append( f'if ({raw_inst}->opsel != 0u || ' f'{raw_inst}->opsel_hi != 3u || ' f'{raw_inst}->opsel_hi_2 != 1u) ' f'[[unlikely]] return emit_error.emit() << "{inst.name} has an invalid ' - 'packed U64 element layout";' + f'{layout_name} element layout";' ) - if inst_sem.semantic_class == 'pk_binop_u64': + if inst_sem.semantic_class in {'pk_binop_f64', 'pk_binop_u64'}: factory_validation_parts.append( f'if ({raw_inst}->src2 != 128u || ' f'({raw_inst}->neg & 4u) != 0u || ' @@ -10421,7 +10435,7 @@ def gen_insts(self) -> None: f'[[unlikely]] return emit_error.emit() << "{inst.name} has an invalid ' 'unused src2 encoding";' ) - else: + if inst_sem.semantic_class == 'pk_lshl_add_u64': factory_validation_parts.append( f'if ({raw_inst}->clamp != 0u || ' f'{raw_inst}->neg != 0u || ' @@ -10449,7 +10463,7 @@ def gen_insts(self) -> None: invalid_sgpr_span = f'{raw_value} > {max_sgpr}u' if opnd.size == 128: # SGPR104_128 is a named VS_128 member: - # s104:s105 provide the scalar U64 value, + # s104:s105 provide the scalar 64-bit value, # while vcc_lo:vcc_hi complete the tuple. invalid_sgpr_span = ( f'({invalid_sgpr_span} && {raw_value} != 104u)' @@ -10509,7 +10523,7 @@ def gen_insts(self) -> None: factory_validation_parts.append( f'if (!({valid_width_specific})) ' f'[[unlikely]] return emit_error.emit() << "{inst.name} has an invalid ' - f'{opnd.name} packed U64 source selector";' + f'{opnd.name} {layout_name} source selector";' ) # Flat segment-aware operands: adjust addr width and add diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py index 74f8ab705a1b..4bab3c3dce05 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/__init__.py @@ -120,6 +120,7 @@ def _register_handlers() -> None: gen_pk_binop_f32, gen_pk_ternary_f32, gen_pk_binop_u64, + gen_pk_binop_f64, gen_pk_lshl_add_u64, gen_pk_mov_b32, gen_mad_mix_f32, @@ -341,6 +342,7 @@ def _register_handlers() -> None: use_cdna5_helpers=c.arch_name == 'cdna5', ) DISPATCH['pk_binop_u64'] = lambda c: gen_pk_binop_u64(c.dst_ops, c.src_ops, c.op) + DISPATCH['pk_binop_f64'] = lambda c: gen_pk_binop_f64(c.dst_ops, c.src_ops, c.op) DISPATCH['pk_lshl_add_u64'] = lambda c: gen_pk_lshl_add_u64(c.dst_ops, c.src_ops) DISPATCH['pk_mov_b32'] = lambda c: gen_pk_mov_b32( c.dst_ops, diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py index f22ac4088b5a..82a488b6cf4a 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/execute/packed.py @@ -734,6 +734,50 @@ def gen_pk_binop_u64(dst: list[str], src: list[str], op: str) -> str: return '\n'.join(L) +def gen_pk_binop_f64(dst: list[str], src: list[str], op: str) -> str: + """Generate two-element packed F64 add/mul/minNum/maxNum execution.""" + operations = { + 'add': 'Add', + 'mul': 'Multiply', + 'max_num': 'MaximumNumber', + 'min_num': 'MinimumNumber', + } + try: + operation = operations[op] + except KeyError as exc: + raise ValueError(f'unsupported packed F64 binary operation: {op}') from exc + + d, s0, s1 = dst[0], src[0], src[1] + return '\n'.join( + [ + ' uint64_t exec = wf.exec();', + ' for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) {', + ' if (!(exec & (1ULL << lane))) continue;', + f' const auto lhs = read_pk_u64_pair({s0}, wf, lane);', + f' const auto rhs = read_pk_u64_pair({s1}, wf, lane);', + ' constexpr uint64_t kSignBit = 0x8000000000000000ULL;', + ' const auto apply = [&](uint64_t lhs_bits, uint64_t rhs_bits,', + ' bool negate_lhs, bool negate_rhs) {', + ' if (negate_lhs) lhs_bits ^= kSignBit;', + ' if (negate_rhs) rhs_bits ^= kSignBit;', + ' uint64_t result = amdgpu::fp_mode::binary_f64(', + f' lhs_bits, rhs_bits, amdgpu::fp_mode::BinaryF64Op::{operation},', + ' wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64());', + ' return amdgpu::fp_mode::finish_f64(', + ' result, wf.fp_round_mode_f16_f64(), 0, inst_.clamp,', + ' amdgpu::floating_clamp_nan_to_zero(wf));', + ' };', + ' const uint64_t result_lo =', + ' apply(lhs.lo, rhs.lo, (inst_.neg & 1u) != 0, (inst_.neg & 2u) != 0);', + ' const uint64_t result_hi = apply(lhs.hi, rhs.hi,', + ' (inst_.neg_hi & 1u) != 0,', + ' (inst_.neg_hi & 2u) != 0);', + f' write_pk_u64_pair({d}, wf, lane, {{result_lo, result_hi}});', + ' }', + ] + ) + + def gen_pk_lshl_add_u64(dst: list[str], src: list[str]) -> str: """Generate packed U64 shift-left-add over two independent elements. diff --git a/emulation/rocjitsu/lib/python/amdisa/semantics.py b/emulation/rocjitsu/lib/python/amdisa/semantics.py index fa7752f07422..24e8a1382be1 100644 --- a/emulation/rocjitsu/lib/python/amdisa/semantics.py +++ b/emulation/rocjitsu/lib/python/amdisa/semantics.py @@ -1603,6 +1603,19 @@ def _derive_vop3p(name: str) -> InstructionSemantics | None: return InstructionSemantics( name, 'pk_binop_f32', operation='add', data_type='f32' ) + packed_f64_binary = { + 'V_PK_ADD_F64': 'add', + 'V_PK_MUL_F64': 'mul', + 'V_PK_MAX_NUM_F64': 'max_num', + 'V_PK_MIN_NUM_F64': 'min_num', + } + if name in packed_f64_binary: + return InstructionSemantics( + name, + 'pk_binop_f64', + operation=packed_f64_binary[name], + data_type='f64', + ) if name in ('V_PK_ADD_NC_U64', 'V_PK_SUB_NC_U64'): operation = 'add' if name == 'V_PK_ADD_NC_U64' else 'sub' return InstructionSemantics( diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py index 745e45fda008..b6d5913d13a6 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_generator_profile_gates.py @@ -5262,6 +5262,78 @@ def test_gfx1251_packed_u64_decode_rejects_undefined_layouts_and_register_tuples assert 'make_after_selector_validation' not in lshl_constructor +def test_gfx1251_packed_f64_decode_rejects_undefined_layouts_and_register_tuples( + gfx1250_generated_root: Path, tmp_path: Path +): + args = SimpleNamespace( + isafiles=[f'cdna5:{_mrisa_dir() / "amdgpu_isa_cdna5.xml"}'], + isa_additions=[f'cdna5:{_mrisa_dir() / "amdgpu_isa_cdna5_gfx1251_delta.xml"}'], + isa_variants=[f'cdna5:{_mrisa_dir() / "amdgpu_isa_cdna5_variants.json"}'], + gen_isas=True, + gen_dbt=False, + isa_output=str(tmp_path), + dbt_output=None, + ) + _run(args) + + for generated_root in (gfx1250_generated_root, tmp_path / 'cdna5'): + source = (generated_root / 'vop3p.cpp').read_text() + for class_name in ( + 'VPkAddF64Vop3p', + 'VPkMulF64Vop3p', + 'VPkMaxNumF64Vop3p', + 'VPkMinNumF64Vop3p', + ): + body = _generated_decode_body(source, class_name) + assert 'has an invalid packed F64 element layout' in body + assert 'has an invalid unused src2 encoding' in body + assert '->neg & 4u) != 0u' in body + assert '->neg_hi & 4u) != 0u' in body + assert 'vdst register tuple that exceeds the selector range' in body + assert 'src0 register tuple that exceeds the selector range' in body + assert 'src1 register tuple that exceeds the selector range' in body + assert 'invalid vdst register tuple alignment' in body + assert 'invalid src0 register tuple alignment' in body + assert 'invalid src1 register tuple alignment' in body + assert 'invalid src0 packed F64 source selector' in body + assert 'invalid src1 packed F64 source selector' in body + for operand_name in ('src0', 'src1'): + assert f'{operand_name} != 104u' in body + assert f'{operand_name} == 104u' in body + assert f'{operand_name} <= 100u' in body + assert f'{operand_name} >= 108u' in body + assert f'{operand_name} <= 120u' in body + assert f'{operand_name} == 124u' in body + assert f'{operand_name} >= 128u' in body + assert f'{operand_name} <= 208u' in body + assert f'{operand_name} >= 240u' in body + assert f'{operand_name} <= 248u' in body + assert f'{operand_name} == 255u' in body + assert f'{operand_name} >= 256u' in body + assert f'{operand_name} <= 508u' in body + for invalid_selector in (230, 231, 235, 236, 253): + assert f'{operand_name} == {invalid_selector}u' not in body + assert 'does not support source modifiers or clamp' not in body + + +def test_gfx1251_packed_f64_literals_use_f64_high_bits_widening( + gfx1250_generated_root: Path, +): + source = (gfx1250_generated_root / 'vop3p.cpp').read_text() + + for class_name in ( + 'VPkFmaF64Vop3p', + 'VPkMulF64Vop3p', + 'VPkAddF64Vop3p', + 'VPkMaxNumF64Vop3p', + 'VPkMinNumF64Vop3p', + ): + constructor = source.split(f'{class_name}::{class_name}(', 1)[1].split( + '\n}', 1 + )[0] + assert 'Operand::Literal32Widening::F64HighBits' in constructor + + def test_gfx1250_matrix_codegen_uses_public_opsel_hi_2_field( gfx1250_generated_root: Path, ): @@ -5348,10 +5420,6 @@ def test_cdna5_variant_execution_callback_inventory( ) -> None: model_only_classes = ( 'VPkFmaF64Vop3p', - 'VPkMulF64Vop3p', - 'VPkAddF64Vop3p', - 'VPkMaxNumF64Vop3p', - 'VPkMinNumF64Vop3p', 'VWmmaF6416x16x4F64Vop3p', ) header = (gfx1250_generated_root / 'vop3p.h').read_text() @@ -5372,8 +5440,12 @@ def test_cdna5_variant_execution_callback_inventory( assert class_name not in execution_source executable_classes = ( + 'VPkMulF64Vop3p', + 'VPkAddF64Vop3p', 'VPkAddNcU64Vop3p', 'VPkSubNcU64Vop3p', + 'VPkMaxNumF64Vop3p', + 'VPkMinNumF64Vop3p', 'VPkLshlAddU64Vop3p', ) for executable_class in executable_classes: diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_literal_codegen.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_literal_codegen.py index 793ab4e503c1..36d87b067a64 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_literal_codegen.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_literal_codegen.py @@ -7,7 +7,7 @@ import pytest -from amdisa.codegen._generator import CodeGenerator +from amdisa.codegen._generator import CodeGenerator, _Literal32Widening from amdisa.codegen.execute.simd_codegen import ( SIMD_VOP2_TERNARY, simd_ternary_literal_operand_name, @@ -272,12 +272,35 @@ def test_f64_simm32_literal_operand_uses_extension_word_as_double_high_bits(): ) == stmt +def test_128bit_packed_f64_widens_simm32_but_packed_u64_does_not(): + packed_f64 = _operand( + 'src0', 'OPR_SRC', size=128, data_format_name='FMT_NUM_PK2_F64' + ) + packed_u64 = _operand( + 'src0', 'OPR_SRC', size=128, data_format_name='FMT_NUM_PK2_U64' + ) + + assert ( + CodeGenerator._literal_operand_simm32_widening( + packed_f64, 'OPR_SIMM32', 'cdna5', 'V_PK_ADD_F64', 'ENC_VOP3P' + ) + is _Literal32Widening.F64_HIGH_BITS + ) + assert ( + CodeGenerator._literal_operand_simm32_widening( + packed_u64, 'OPR_SIMM32', 'cdna5', 'V_PK_ADD_U64', 'ENC_VOP3P' + ) + is None + ) + + def test_64bit_simm32_literal_operand_requires_data_format(): with pytest.raises( ValueError, match=( "architecture 'rdna4', instruction 'V_FMAC_F64', encoding 'ENC_VOP2': " - "64-bit SIMM32 input operand 'src0' has unsupported data format ''" + "SIMM32 input operand 'src0' has no widening policy for size 64 and " + "data format ''" ), ): CodeGenerator._literal_operand_fixup_stmt( @@ -400,7 +423,8 @@ def test_64bit_simm32_literal_operand_rejects_unrecognized_data_format(): ValueError, match=( "architecture 'cdna4', instruction 'S_FAKE_I64', encoding 'ENC_SOP2': " - "64-bit SIMM32 input operand 'ssrc0' has unsupported data format 'FMT_NUM_X64'" + "SIMM32 input operand 'ssrc0' has no widening policy for size 64 and " + "data format 'FMT_NUM_X64'" ), ): CodeGenerator._literal_operand_fixup_stmt( diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py index a0e766cf843e..92325fbf5b34 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_packed_codegen.py @@ -13,6 +13,7 @@ gen_mad_mix_lo_hi, gen_pk_binop, gen_pk_binop_f32, + gen_pk_binop_f64, gen_pk_fmac_vop2, gen_pk_fmac_vop3, gen_pk_binop_u64, @@ -394,6 +395,36 @@ def test_pk_u64_binop_rejects_unknown_operation(): gen_pk_binop_u64(['vdst'], ['src0', 'src1'], 'mul') +def test_pk_f64_binops_use_mode_aware_two_element_execution(): + expected_operations = { + 'add': 'Add', + 'mul': 'Multiply', + 'max_num': 'MaximumNumber', + 'min_num': 'MinimumNumber', + } + for operation, enum_name in expected_operations.items(): + cpp = gen_pk_binop_f64(['vdst'], ['src0', 'src1'], operation) + + assert 'const auto lhs = read_pk_u64_pair(src0, wf, lane);' in cpp + assert 'const auto rhs = read_pk_u64_pair(src1, wf, lane);' in cpp + assert f'BinaryF64Op::{enum_name}' in cpp + assert cpp.count('amdgpu::fp_mode::binary_f64(') == 1 + assert 'wf.fp_round_mode_f16_f64()' in cpp + assert 'wf.fp_denorm_mode_f16_f64()' in cpp + assert 'lhs_bits ^= kSignBit' in cpp + assert 'rhs_bits ^= kSignBit' in cpp + assert 'amdgpu::fp_mode::finish_f64(' in cpp + assert 'inst_.clamp' in cpp + assert 'write_pk_u64_pair(vdst, wf, lane, {result_lo, result_hi});' in cpp + + +def test_pk_f64_binop_rejects_unknown_operation(): + import pytest + + with pytest.raises(ValueError, match='unsupported packed F64 binary operation'): + gen_pk_binop_f64(['vdst'], ['src0', 'src1'], 'divide') + + def test_renamed_vop3p_packed_f32_probe_passes_profile_selectors(): probe = vop3p_local_simd_probe_line('v_pk_add_f32_vop3p', ('opsel', 'opsel_hi')) diff --git a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py index a1fe479377e1..1e78a4149a0f 100644 --- a/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py +++ b/emulation/rocjitsu/lib/python/amdisa/tests/test_sema_derive.py @@ -2661,6 +2661,20 @@ def test_pk_u64_add_sub_have_distinct_binary_semantics(self): assert sem.operation == operation assert sem.data_type == 'u64' + def test_pk_f64_basic_arithmetic_has_distinct_binary_semantics(self): + for name, operation in ( + ('V_PK_ADD_F64', 'add'), + ('V_PK_MUL_F64', 'mul'), + ('V_PK_MAX_NUM_F64', 'max_num'), + ('V_PK_MIN_NUM_F64', 'min_num'), + ): + sem = derive_semantics(name, 'ENC_VOP3P') + + assert sem is not None + assert sem.semantic_class == 'pk_binop_f64' + assert sem.operation == operation + assert sem.data_type == 'f64' + def test_pk_mov_b32(self): sem = _FakeSem('V_PK_MOV_B32', 'pk_mov_b32') block = derive_sema_block(sem) diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/execution_backend.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/execution_backend.h index 93e8e8577ec7..5cb50cdad650 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/execution_backend.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/execution_backend.h @@ -1269,8 +1269,12 @@ enum class InstructionExecutionId : size_t { VWmmaF1616x16x128Bf8Fp8Vop3p, VWmmaF1616x16x128Bf8Bf8Vop3p, VWmmaF3232x16x128F4Vop3p, + VPkMulF64Vop3p, + VPkAddF64Vop3p, VPkAddNcU64Vop3p, VPkSubNcU64Vop3p, + VPkMaxNumF64Vop3p, + VPkMinNumF64Vop3p, VPkLshlAddU64Vop3p, VWmmaScaleF32Vop3px2, DsAddU32Vds, diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/execution_backend_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/execution_backend_exec.cpp index 470034d263de..6666a0be8994 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/execution_backend_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/execution_backend_exec.cpp @@ -1276,8 +1276,12 @@ constexpr InstructionCallbackTable kInstructionCallbacks{{ &execute_with_backend, &execute_with_backend, &execute_with_backend, + &execute_with_backend, + &execute_with_backend, &execute_with_backend, &execute_with_backend, + &execute_with_backend, + &execute_with_backend, &execute_with_backend, &execute_with_backend, &execute_with_backend, diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.cpp index ad19e83e9dbd..80b4c3ace7f2 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.cpp @@ -4183,17 +4183,20 @@ VPkFmaF64Vop3p::VPkFmaF64Vop3p(const MachineInst *inst) num_src_ = 3; num_dst_ = 1; if (reinterpret_cast(inst)->src0 == 255) - src0 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src0 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); if (reinterpret_cast(inst)->src1 == 255) - src1 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src1 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); if (reinterpret_cast(inst)->src2 == 255) - src2 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src2 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); vdst.set_vgpr_msb_role(amdgpu::VgprMsbRole::Dst); src0.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src0); src1.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src1); @@ -4217,7 +4220,8 @@ DecodeResult decodeVPkFmaF64Vop3p(const MachineInst *opcode, const DecodeErrorEm } // namespace detail VPkMulF64Vop3p::VPkMulF64Vop3p(const MachineInst *inst) - : Vop3p("v_pk_mul_f64", reinterpret_cast(inst), nullptr), + : Vop3p("v_pk_mul_f64", reinterpret_cast(inst), + selected_exec_fn(InstructionExecutionId::VPkMulF64Vop3p)), vdst(128, OperandType::OPR_VGPR, reinterpret_cast(inst)->vdst), src0(128, OperandType::OPR_SRC, reinterpret_cast(inst)->src0), src1(128, OperandType::OPR_SRC, reinterpret_cast(inst)->src1) { @@ -4228,13 +4232,15 @@ VPkMulF64Vop3p::VPkMulF64Vop3p(const MachineInst *inst) num_src_ = 2; num_dst_ = 1; if (reinterpret_cast(inst)->src0 == 255) - src0 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src0 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); if (reinterpret_cast(inst)->src1 == 255) - src1 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src1 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); vdst.set_vgpr_msb_role(amdgpu::VgprMsbRole::Dst); src0.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src0); src1.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src1); @@ -4248,6 +4254,81 @@ DecodeResult decodeVPkMulF64Vop3p(const MachineInst *opcode, const DecodeErrorEm emit_error, LiteralSupport::Literal32, 2); if (validation.failed()) [[unlikely]] return Result::failure(); + if (reinterpret_cast(inst)->opsel != 0u || + reinterpret_cast(inst)->opsel_hi != 3u || + reinterpret_cast(inst)->opsel_hi_2 != 1u) [[unlikely]] + return emit_error.emit() << "V_PK_MUL_F64 has an invalid packed F64 element layout"; + if (reinterpret_cast(inst)->src2 != 128u || + (reinterpret_cast(inst)->neg & 4u) != 0u || + (reinterpret_cast(inst)->neg_hi & 4u) != 0u) [[unlikely]] + return emit_error.emit() << "V_PK_MUL_F64 has an invalid unused src2 encoding"; + if (reinterpret_cast(inst)->vdst > 252u) [[unlikely]] + return emit_error.emit() + << "V_PK_MUL_F64 has a vdst register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->vdst & 1u) != 0u) [[unlikely]] + return emit_error.emit() << "V_PK_MUL_F64 has an invalid vdst register tuple alignment"; + if ((reinterpret_cast(inst)->src0 <= 105u && + (reinterpret_cast(inst)->src0 > 102u && + reinterpret_cast(inst)->src0 != 104u)) || + (reinterpret_cast(inst)->src0 >= 256u && + reinterpret_cast(inst)->src0 > 508u)) [[unlikely]] + return emit_error.emit() + << "V_PK_MUL_F64 has a src0 register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->src0 <= 105u && + (reinterpret_cast(inst)->src0 % 4u) != 0u) || + (reinterpret_cast(inst)->src0 >= 108u && + reinterpret_cast(inst)->src0 <= 123u && + ((reinterpret_cast(inst)->src0 - 108u) % 4u) != 0u) || + (reinterpret_cast(inst)->src0 >= 256u && + (reinterpret_cast(inst)->src0 & 1u) != 0u)) [[unlikely]] + return emit_error.emit() << "V_PK_MUL_F64 has an invalid src0 register tuple alignment"; + if (!((reinterpret_cast(inst)->src0 <= 100u && + (reinterpret_cast(inst)->src0 % 4u) == 0u) || + reinterpret_cast(inst)->src0 == 104u || + (reinterpret_cast(inst)->src0 >= 108u && + reinterpret_cast(inst)->src0 <= 120u && + ((reinterpret_cast(inst)->src0 - 108u) % 4u) == 0u) || + reinterpret_cast(inst)->src0 == 124u || + (reinterpret_cast(inst)->src0 >= 128u && + reinterpret_cast(inst)->src0 <= 208u) || + (reinterpret_cast(inst)->src0 >= 240u && + reinterpret_cast(inst)->src0 <= 248u) || + reinterpret_cast(inst)->src0 == 255u || + (reinterpret_cast(inst)->src0 >= 256u && + reinterpret_cast(inst)->src0 <= 508u && + (reinterpret_cast(inst)->src0 & 1u) == 0u))) [[unlikely]] + return emit_error.emit() << "V_PK_MUL_F64 has an invalid src0 packed F64 source selector"; + if ((reinterpret_cast(inst)->src1 <= 105u && + (reinterpret_cast(inst)->src1 > 102u && + reinterpret_cast(inst)->src1 != 104u)) || + (reinterpret_cast(inst)->src1 >= 256u && + reinterpret_cast(inst)->src1 > 508u)) [[unlikely]] + return emit_error.emit() + << "V_PK_MUL_F64 has a src1 register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->src1 <= 105u && + (reinterpret_cast(inst)->src1 % 4u) != 0u) || + (reinterpret_cast(inst)->src1 >= 108u && + reinterpret_cast(inst)->src1 <= 123u && + ((reinterpret_cast(inst)->src1 - 108u) % 4u) != 0u) || + (reinterpret_cast(inst)->src1 >= 256u && + (reinterpret_cast(inst)->src1 & 1u) != 0u)) [[unlikely]] + return emit_error.emit() << "V_PK_MUL_F64 has an invalid src1 register tuple alignment"; + if (!((reinterpret_cast(inst)->src1 <= 100u && + (reinterpret_cast(inst)->src1 % 4u) == 0u) || + reinterpret_cast(inst)->src1 == 104u || + (reinterpret_cast(inst)->src1 >= 108u && + reinterpret_cast(inst)->src1 <= 120u && + ((reinterpret_cast(inst)->src1 - 108u) % 4u) == 0u) || + reinterpret_cast(inst)->src1 == 124u || + (reinterpret_cast(inst)->src1 >= 128u && + reinterpret_cast(inst)->src1 <= 208u) || + (reinterpret_cast(inst)->src1 >= 240u && + reinterpret_cast(inst)->src1 <= 248u) || + reinterpret_cast(inst)->src1 == 255u || + (reinterpret_cast(inst)->src1 >= 256u && + reinterpret_cast(inst)->src1 <= 508u && + (reinterpret_cast(inst)->src1 & 1u) == 0u))) [[unlikely]] + return emit_error.emit() << "V_PK_MUL_F64 has an invalid src1 packed F64 source selector"; if (reinterpret_cast(inst)->src0 == amdgpu::SRC_DPP || amdgpu::dpp::is_src_dpp8(reinterpret_cast(inst)->src0)) [[unlikely]] @@ -4257,7 +4338,8 @@ DecodeResult decodeVPkMulF64Vop3p(const MachineInst *opcode, const DecodeErrorEm } // namespace detail VPkAddF64Vop3p::VPkAddF64Vop3p(const MachineInst *inst) - : Vop3p("v_pk_add_f64", reinterpret_cast(inst), nullptr), + : Vop3p("v_pk_add_f64", reinterpret_cast(inst), + selected_exec_fn(InstructionExecutionId::VPkAddF64Vop3p)), vdst(128, OperandType::OPR_VGPR, reinterpret_cast(inst)->vdst), src0(128, OperandType::OPR_SRC, reinterpret_cast(inst)->src0), src1(128, OperandType::OPR_SRC, reinterpret_cast(inst)->src1) { @@ -4268,13 +4350,15 @@ VPkAddF64Vop3p::VPkAddF64Vop3p(const MachineInst *inst) num_src_ = 2; num_dst_ = 1; if (reinterpret_cast(inst)->src0 == 255) - src0 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src0 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); if (reinterpret_cast(inst)->src1 == 255) - src1 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src1 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); vdst.set_vgpr_msb_role(amdgpu::VgprMsbRole::Dst); src0.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src0); src1.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src1); @@ -4288,6 +4372,81 @@ DecodeResult decodeVPkAddF64Vop3p(const MachineInst *opcode, const DecodeErrorEm emit_error, LiteralSupport::Literal32, 2); if (validation.failed()) [[unlikely]] return Result::failure(); + if (reinterpret_cast(inst)->opsel != 0u || + reinterpret_cast(inst)->opsel_hi != 3u || + reinterpret_cast(inst)->opsel_hi_2 != 1u) [[unlikely]] + return emit_error.emit() << "V_PK_ADD_F64 has an invalid packed F64 element layout"; + if (reinterpret_cast(inst)->src2 != 128u || + (reinterpret_cast(inst)->neg & 4u) != 0u || + (reinterpret_cast(inst)->neg_hi & 4u) != 0u) [[unlikely]] + return emit_error.emit() << "V_PK_ADD_F64 has an invalid unused src2 encoding"; + if (reinterpret_cast(inst)->vdst > 252u) [[unlikely]] + return emit_error.emit() + << "V_PK_ADD_F64 has a vdst register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->vdst & 1u) != 0u) [[unlikely]] + return emit_error.emit() << "V_PK_ADD_F64 has an invalid vdst register tuple alignment"; + if ((reinterpret_cast(inst)->src0 <= 105u && + (reinterpret_cast(inst)->src0 > 102u && + reinterpret_cast(inst)->src0 != 104u)) || + (reinterpret_cast(inst)->src0 >= 256u && + reinterpret_cast(inst)->src0 > 508u)) [[unlikely]] + return emit_error.emit() + << "V_PK_ADD_F64 has a src0 register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->src0 <= 105u && + (reinterpret_cast(inst)->src0 % 4u) != 0u) || + (reinterpret_cast(inst)->src0 >= 108u && + reinterpret_cast(inst)->src0 <= 123u && + ((reinterpret_cast(inst)->src0 - 108u) % 4u) != 0u) || + (reinterpret_cast(inst)->src0 >= 256u && + (reinterpret_cast(inst)->src0 & 1u) != 0u)) [[unlikely]] + return emit_error.emit() << "V_PK_ADD_F64 has an invalid src0 register tuple alignment"; + if (!((reinterpret_cast(inst)->src0 <= 100u && + (reinterpret_cast(inst)->src0 % 4u) == 0u) || + reinterpret_cast(inst)->src0 == 104u || + (reinterpret_cast(inst)->src0 >= 108u && + reinterpret_cast(inst)->src0 <= 120u && + ((reinterpret_cast(inst)->src0 - 108u) % 4u) == 0u) || + reinterpret_cast(inst)->src0 == 124u || + (reinterpret_cast(inst)->src0 >= 128u && + reinterpret_cast(inst)->src0 <= 208u) || + (reinterpret_cast(inst)->src0 >= 240u && + reinterpret_cast(inst)->src0 <= 248u) || + reinterpret_cast(inst)->src0 == 255u || + (reinterpret_cast(inst)->src0 >= 256u && + reinterpret_cast(inst)->src0 <= 508u && + (reinterpret_cast(inst)->src0 & 1u) == 0u))) [[unlikely]] + return emit_error.emit() << "V_PK_ADD_F64 has an invalid src0 packed F64 source selector"; + if ((reinterpret_cast(inst)->src1 <= 105u && + (reinterpret_cast(inst)->src1 > 102u && + reinterpret_cast(inst)->src1 != 104u)) || + (reinterpret_cast(inst)->src1 >= 256u && + reinterpret_cast(inst)->src1 > 508u)) [[unlikely]] + return emit_error.emit() + << "V_PK_ADD_F64 has a src1 register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->src1 <= 105u && + (reinterpret_cast(inst)->src1 % 4u) != 0u) || + (reinterpret_cast(inst)->src1 >= 108u && + reinterpret_cast(inst)->src1 <= 123u && + ((reinterpret_cast(inst)->src1 - 108u) % 4u) != 0u) || + (reinterpret_cast(inst)->src1 >= 256u && + (reinterpret_cast(inst)->src1 & 1u) != 0u)) [[unlikely]] + return emit_error.emit() << "V_PK_ADD_F64 has an invalid src1 register tuple alignment"; + if (!((reinterpret_cast(inst)->src1 <= 100u && + (reinterpret_cast(inst)->src1 % 4u) == 0u) || + reinterpret_cast(inst)->src1 == 104u || + (reinterpret_cast(inst)->src1 >= 108u && + reinterpret_cast(inst)->src1 <= 120u && + ((reinterpret_cast(inst)->src1 - 108u) % 4u) == 0u) || + reinterpret_cast(inst)->src1 == 124u || + (reinterpret_cast(inst)->src1 >= 128u && + reinterpret_cast(inst)->src1 <= 208u) || + (reinterpret_cast(inst)->src1 >= 240u && + reinterpret_cast(inst)->src1 <= 248u) || + reinterpret_cast(inst)->src1 == 255u || + (reinterpret_cast(inst)->src1 >= 256u && + reinterpret_cast(inst)->src1 <= 508u && + (reinterpret_cast(inst)->src1 & 1u) == 0u))) [[unlikely]] + return emit_error.emit() << "V_PK_ADD_F64 has an invalid src1 packed F64 source selector"; if (reinterpret_cast(inst)->src0 == amdgpu::SRC_DPP || amdgpu::dpp::is_src_dpp8(reinterpret_cast(inst)->src0)) [[unlikely]] @@ -4531,7 +4690,8 @@ DecodeResult decodeVPkSubNcU64Vop3p(const MachineInst *opcode, } // namespace detail VPkMaxNumF64Vop3p::VPkMaxNumF64Vop3p(const MachineInst *inst) - : Vop3p("v_pk_max_num_f64", reinterpret_cast(inst), nullptr), + : Vop3p("v_pk_max_num_f64", reinterpret_cast(inst), + selected_exec_fn(InstructionExecutionId::VPkMaxNumF64Vop3p)), vdst(128, OperandType::OPR_VGPR, reinterpret_cast(inst)->vdst), src0(128, OperandType::OPR_SRC, reinterpret_cast(inst)->src0), src1(128, OperandType::OPR_SRC, reinterpret_cast(inst)->src1) { @@ -4542,13 +4702,15 @@ VPkMaxNumF64Vop3p::VPkMaxNumF64Vop3p(const MachineInst *inst) num_src_ = 2; num_dst_ = 1; if (reinterpret_cast(inst)->src0 == 255) - src0 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src0 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); if (reinterpret_cast(inst)->src1 == 255) - src1 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src1 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); vdst.set_vgpr_msb_role(amdgpu::VgprMsbRole::Dst); src0.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src0); src1.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src1); @@ -4563,6 +4725,81 @@ DecodeResult decodeVPkMaxNumF64Vop3p(const MachineInst *opcode, emit_error, LiteralSupport::Literal32, 2); if (validation.failed()) [[unlikely]] return Result::failure(); + if (reinterpret_cast(inst)->opsel != 0u || + reinterpret_cast(inst)->opsel_hi != 3u || + reinterpret_cast(inst)->opsel_hi_2 != 1u) [[unlikely]] + return emit_error.emit() << "V_PK_MAX_NUM_F64 has an invalid packed F64 element layout"; + if (reinterpret_cast(inst)->src2 != 128u || + (reinterpret_cast(inst)->neg & 4u) != 0u || + (reinterpret_cast(inst)->neg_hi & 4u) != 0u) [[unlikely]] + return emit_error.emit() << "V_PK_MAX_NUM_F64 has an invalid unused src2 encoding"; + if (reinterpret_cast(inst)->vdst > 252u) [[unlikely]] + return emit_error.emit() + << "V_PK_MAX_NUM_F64 has a vdst register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->vdst & 1u) != 0u) [[unlikely]] + return emit_error.emit() << "V_PK_MAX_NUM_F64 has an invalid vdst register tuple alignment"; + if ((reinterpret_cast(inst)->src0 <= 105u && + (reinterpret_cast(inst)->src0 > 102u && + reinterpret_cast(inst)->src0 != 104u)) || + (reinterpret_cast(inst)->src0 >= 256u && + reinterpret_cast(inst)->src0 > 508u)) [[unlikely]] + return emit_error.emit() + << "V_PK_MAX_NUM_F64 has a src0 register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->src0 <= 105u && + (reinterpret_cast(inst)->src0 % 4u) != 0u) || + (reinterpret_cast(inst)->src0 >= 108u && + reinterpret_cast(inst)->src0 <= 123u && + ((reinterpret_cast(inst)->src0 - 108u) % 4u) != 0u) || + (reinterpret_cast(inst)->src0 >= 256u && + (reinterpret_cast(inst)->src0 & 1u) != 0u)) [[unlikely]] + return emit_error.emit() << "V_PK_MAX_NUM_F64 has an invalid src0 register tuple alignment"; + if (!((reinterpret_cast(inst)->src0 <= 100u && + (reinterpret_cast(inst)->src0 % 4u) == 0u) || + reinterpret_cast(inst)->src0 == 104u || + (reinterpret_cast(inst)->src0 >= 108u && + reinterpret_cast(inst)->src0 <= 120u && + ((reinterpret_cast(inst)->src0 - 108u) % 4u) == 0u) || + reinterpret_cast(inst)->src0 == 124u || + (reinterpret_cast(inst)->src0 >= 128u && + reinterpret_cast(inst)->src0 <= 208u) || + (reinterpret_cast(inst)->src0 >= 240u && + reinterpret_cast(inst)->src0 <= 248u) || + reinterpret_cast(inst)->src0 == 255u || + (reinterpret_cast(inst)->src0 >= 256u && + reinterpret_cast(inst)->src0 <= 508u && + (reinterpret_cast(inst)->src0 & 1u) == 0u))) [[unlikely]] + return emit_error.emit() << "V_PK_MAX_NUM_F64 has an invalid src0 packed F64 source selector"; + if ((reinterpret_cast(inst)->src1 <= 105u && + (reinterpret_cast(inst)->src1 > 102u && + reinterpret_cast(inst)->src1 != 104u)) || + (reinterpret_cast(inst)->src1 >= 256u && + reinterpret_cast(inst)->src1 > 508u)) [[unlikely]] + return emit_error.emit() + << "V_PK_MAX_NUM_F64 has a src1 register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->src1 <= 105u && + (reinterpret_cast(inst)->src1 % 4u) != 0u) || + (reinterpret_cast(inst)->src1 >= 108u && + reinterpret_cast(inst)->src1 <= 123u && + ((reinterpret_cast(inst)->src1 - 108u) % 4u) != 0u) || + (reinterpret_cast(inst)->src1 >= 256u && + (reinterpret_cast(inst)->src1 & 1u) != 0u)) [[unlikely]] + return emit_error.emit() << "V_PK_MAX_NUM_F64 has an invalid src1 register tuple alignment"; + if (!((reinterpret_cast(inst)->src1 <= 100u && + (reinterpret_cast(inst)->src1 % 4u) == 0u) || + reinterpret_cast(inst)->src1 == 104u || + (reinterpret_cast(inst)->src1 >= 108u && + reinterpret_cast(inst)->src1 <= 120u && + ((reinterpret_cast(inst)->src1 - 108u) % 4u) == 0u) || + reinterpret_cast(inst)->src1 == 124u || + (reinterpret_cast(inst)->src1 >= 128u && + reinterpret_cast(inst)->src1 <= 208u) || + (reinterpret_cast(inst)->src1 >= 240u && + reinterpret_cast(inst)->src1 <= 248u) || + reinterpret_cast(inst)->src1 == 255u || + (reinterpret_cast(inst)->src1 >= 256u && + reinterpret_cast(inst)->src1 <= 508u && + (reinterpret_cast(inst)->src1 & 1u) == 0u))) [[unlikely]] + return emit_error.emit() << "V_PK_MAX_NUM_F64 has an invalid src1 packed F64 source selector"; if (reinterpret_cast(inst)->src0 == amdgpu::SRC_DPP || amdgpu::dpp::is_src_dpp8(reinterpret_cast(inst)->src0)) [[unlikely]] @@ -4572,7 +4809,8 @@ DecodeResult decodeVPkMaxNumF64Vop3p(const MachineInst *opcode, } // namespace detail VPkMinNumF64Vop3p::VPkMinNumF64Vop3p(const MachineInst *inst) - : Vop3p("v_pk_min_num_f64", reinterpret_cast(inst), nullptr), + : Vop3p("v_pk_min_num_f64", reinterpret_cast(inst), + selected_exec_fn(InstructionExecutionId::VPkMinNumF64Vop3p)), vdst(128, OperandType::OPR_VGPR, reinterpret_cast(inst)->vdst), src0(128, OperandType::OPR_SRC, reinterpret_cast(inst)->src0), src1(128, OperandType::OPR_SRC, reinterpret_cast(inst)->src1) { @@ -4583,13 +4821,15 @@ VPkMinNumF64Vop3p::VPkMinNumF64Vop3p(const MachineInst *inst) num_src_ = 2; num_dst_ = 1; if (reinterpret_cast(inst)->src0 == 255) - src0 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src0 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); if (reinterpret_cast(inst)->src1 == 255) - src1 = Operand( - 128, OperandType::OPR_SIMM32, - static_cast(reinterpret_cast(inst)->simm32)); + src1 = Operand::make_literal32( + 128, + static_cast(reinterpret_cast(inst)->simm32), + Operand::Literal32Widening::F64HighBits); vdst.set_vgpr_msb_role(amdgpu::VgprMsbRole::Dst); src0.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src0); src1.set_vgpr_msb_role(amdgpu::VgprMsbRole::Src1); @@ -4604,6 +4844,81 @@ DecodeResult decodeVPkMinNumF64Vop3p(const MachineInst *opcode, emit_error, LiteralSupport::Literal32, 2); if (validation.failed()) [[unlikely]] return Result::failure(); + if (reinterpret_cast(inst)->opsel != 0u || + reinterpret_cast(inst)->opsel_hi != 3u || + reinterpret_cast(inst)->opsel_hi_2 != 1u) [[unlikely]] + return emit_error.emit() << "V_PK_MIN_NUM_F64 has an invalid packed F64 element layout"; + if (reinterpret_cast(inst)->src2 != 128u || + (reinterpret_cast(inst)->neg & 4u) != 0u || + (reinterpret_cast(inst)->neg_hi & 4u) != 0u) [[unlikely]] + return emit_error.emit() << "V_PK_MIN_NUM_F64 has an invalid unused src2 encoding"; + if (reinterpret_cast(inst)->vdst > 252u) [[unlikely]] + return emit_error.emit() + << "V_PK_MIN_NUM_F64 has a vdst register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->vdst & 1u) != 0u) [[unlikely]] + return emit_error.emit() << "V_PK_MIN_NUM_F64 has an invalid vdst register tuple alignment"; + if ((reinterpret_cast(inst)->src0 <= 105u && + (reinterpret_cast(inst)->src0 > 102u && + reinterpret_cast(inst)->src0 != 104u)) || + (reinterpret_cast(inst)->src0 >= 256u && + reinterpret_cast(inst)->src0 > 508u)) [[unlikely]] + return emit_error.emit() + << "V_PK_MIN_NUM_F64 has a src0 register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->src0 <= 105u && + (reinterpret_cast(inst)->src0 % 4u) != 0u) || + (reinterpret_cast(inst)->src0 >= 108u && + reinterpret_cast(inst)->src0 <= 123u && + ((reinterpret_cast(inst)->src0 - 108u) % 4u) != 0u) || + (reinterpret_cast(inst)->src0 >= 256u && + (reinterpret_cast(inst)->src0 & 1u) != 0u)) [[unlikely]] + return emit_error.emit() << "V_PK_MIN_NUM_F64 has an invalid src0 register tuple alignment"; + if (!((reinterpret_cast(inst)->src0 <= 100u && + (reinterpret_cast(inst)->src0 % 4u) == 0u) || + reinterpret_cast(inst)->src0 == 104u || + (reinterpret_cast(inst)->src0 >= 108u && + reinterpret_cast(inst)->src0 <= 120u && + ((reinterpret_cast(inst)->src0 - 108u) % 4u) == 0u) || + reinterpret_cast(inst)->src0 == 124u || + (reinterpret_cast(inst)->src0 >= 128u && + reinterpret_cast(inst)->src0 <= 208u) || + (reinterpret_cast(inst)->src0 >= 240u && + reinterpret_cast(inst)->src0 <= 248u) || + reinterpret_cast(inst)->src0 == 255u || + (reinterpret_cast(inst)->src0 >= 256u && + reinterpret_cast(inst)->src0 <= 508u && + (reinterpret_cast(inst)->src0 & 1u) == 0u))) [[unlikely]] + return emit_error.emit() << "V_PK_MIN_NUM_F64 has an invalid src0 packed F64 source selector"; + if ((reinterpret_cast(inst)->src1 <= 105u && + (reinterpret_cast(inst)->src1 > 102u && + reinterpret_cast(inst)->src1 != 104u)) || + (reinterpret_cast(inst)->src1 >= 256u && + reinterpret_cast(inst)->src1 > 508u)) [[unlikely]] + return emit_error.emit() + << "V_PK_MIN_NUM_F64 has a src1 register tuple that exceeds the selector range"; + if ((reinterpret_cast(inst)->src1 <= 105u && + (reinterpret_cast(inst)->src1 % 4u) != 0u) || + (reinterpret_cast(inst)->src1 >= 108u && + reinterpret_cast(inst)->src1 <= 123u && + ((reinterpret_cast(inst)->src1 - 108u) % 4u) != 0u) || + (reinterpret_cast(inst)->src1 >= 256u && + (reinterpret_cast(inst)->src1 & 1u) != 0u)) [[unlikely]] + return emit_error.emit() << "V_PK_MIN_NUM_F64 has an invalid src1 register tuple alignment"; + if (!((reinterpret_cast(inst)->src1 <= 100u && + (reinterpret_cast(inst)->src1 % 4u) == 0u) || + reinterpret_cast(inst)->src1 == 104u || + (reinterpret_cast(inst)->src1 >= 108u && + reinterpret_cast(inst)->src1 <= 120u && + ((reinterpret_cast(inst)->src1 - 108u) % 4u) == 0u) || + reinterpret_cast(inst)->src1 == 124u || + (reinterpret_cast(inst)->src1 >= 128u && + reinterpret_cast(inst)->src1 <= 208u) || + (reinterpret_cast(inst)->src1 >= 240u && + reinterpret_cast(inst)->src1 <= 248u) || + reinterpret_cast(inst)->src1 == 255u || + (reinterpret_cast(inst)->src1 >= 256u && + reinterpret_cast(inst)->src1 <= 508u && + (reinterpret_cast(inst)->src1 & 1u) == 0u))) [[unlikely]] + return emit_error.emit() << "V_PK_MIN_NUM_F64 has an invalid src1 packed F64 source selector"; if (reinterpret_cast(inst)->src0 == amdgpu::SRC_DPP || amdgpu::dpp::is_src_dpp8(reinterpret_cast(inst)->src0)) [[unlikely]] diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.h index e9ecdd97dd9c..621a37962fd7 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p.h @@ -944,6 +944,7 @@ class VPkFmaF64Vop3p : public Vop3p { class VPkMulF64Vop3p : public Vop3p { public: VPkMulF64Vop3p(const MachineInst *inst); + void execute_impl(amdgpu::Wavefront &wf); Operand vdst; Operand src0; Operand src1; @@ -952,6 +953,7 @@ class VPkMulF64Vop3p : public Vop3p { class VPkAddF64Vop3p : public Vop3p { public: VPkAddF64Vop3p(const MachineInst *inst); + void execute_impl(amdgpu::Wavefront &wf); Operand vdst; Operand src0; Operand src1; @@ -978,6 +980,7 @@ class VPkSubNcU64Vop3p : public Vop3p { class VPkMaxNumF64Vop3p : public Vop3p { public: VPkMaxNumF64Vop3p(const MachineInst *inst); + void execute_impl(amdgpu::Wavefront &wf); Operand vdst; Operand src0; Operand src1; @@ -986,6 +989,7 @@ class VPkMaxNumF64Vop3p : public Vop3p { class VPkMinNumF64Vop3p : public Vop3p { public: VPkMinNumF64Vop3p(const MachineInst *inst); + void execute_impl(amdgpu::Wavefront &wf); Operand vdst; Operand src0; Operand src1; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp index d0e92992b0d9..582f22f2242a 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/vop3p_exec.cpp @@ -2918,6 +2918,58 @@ void VWmmaF3232x16x128F4Vop3p::execute_impl(amdgpu::Wavefront &wf) { amdgpu::wmma_c_modifier(inst_.neg, inst_.neg_hi)); } +void VPkMulF64Vop3p::execute_impl(amdgpu::Wavefront &wf) { + uint64_t exec = wf.exec(); + for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { + if (!(exec & (1ULL << lane))) + continue; + const auto lhs = read_pk_u64_pair(src0, wf, lane); + const auto rhs = read_pk_u64_pair(src1, wf, lane); + constexpr uint64_t kSignBit = 0x8000000000000000ULL; + const auto apply = [&](uint64_t lhs_bits, uint64_t rhs_bits, bool negate_lhs, bool negate_rhs) { + if (negate_lhs) + lhs_bits ^= kSignBit; + if (negate_rhs) + rhs_bits ^= kSignBit; + uint64_t result = + amdgpu::fp_mode::binary_f64(lhs_bits, rhs_bits, amdgpu::fp_mode::BinaryF64Op::Multiply, + wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64()); + return amdgpu::fp_mode::finish_f64(result, wf.fp_round_mode_f16_f64(), 0, inst_.clamp, + amdgpu::floating_clamp_nan_to_zero(wf)); + }; + const uint64_t result_lo = apply(lhs.lo, rhs.lo, (inst_.neg & 1u) != 0, (inst_.neg & 2u) != 0); + const uint64_t result_hi = + apply(lhs.hi, rhs.hi, (inst_.neg_hi & 1u) != 0, (inst_.neg_hi & 2u) != 0); + write_pk_u64_pair(vdst, wf, lane, {result_lo, result_hi}); + } +} + +void VPkAddF64Vop3p::execute_impl(amdgpu::Wavefront &wf) { + uint64_t exec = wf.exec(); + for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { + if (!(exec & (1ULL << lane))) + continue; + const auto lhs = read_pk_u64_pair(src0, wf, lane); + const auto rhs = read_pk_u64_pair(src1, wf, lane); + constexpr uint64_t kSignBit = 0x8000000000000000ULL; + const auto apply = [&](uint64_t lhs_bits, uint64_t rhs_bits, bool negate_lhs, bool negate_rhs) { + if (negate_lhs) + lhs_bits ^= kSignBit; + if (negate_rhs) + rhs_bits ^= kSignBit; + uint64_t result = + amdgpu::fp_mode::binary_f64(lhs_bits, rhs_bits, amdgpu::fp_mode::BinaryF64Op::Add, + wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64()); + return amdgpu::fp_mode::finish_f64(result, wf.fp_round_mode_f16_f64(), 0, inst_.clamp, + amdgpu::floating_clamp_nan_to_zero(wf)); + }; + const uint64_t result_lo = apply(lhs.lo, rhs.lo, (inst_.neg & 1u) != 0, (inst_.neg & 2u) != 0); + const uint64_t result_hi = + apply(lhs.hi, rhs.hi, (inst_.neg_hi & 1u) != 0, (inst_.neg_hi & 2u) != 0); + write_pk_u64_pair(vdst, wf, lane, {result_lo, result_hi}); + } +} + void VPkAddNcU64Vop3p::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { @@ -2984,6 +3036,58 @@ void VPkSubNcU64Vop3p::execute_impl(amdgpu::Wavefront &wf) { } } +void VPkMaxNumF64Vop3p::execute_impl(amdgpu::Wavefront &wf) { + uint64_t exec = wf.exec(); + for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { + if (!(exec & (1ULL << lane))) + continue; + const auto lhs = read_pk_u64_pair(src0, wf, lane); + const auto rhs = read_pk_u64_pair(src1, wf, lane); + constexpr uint64_t kSignBit = 0x8000000000000000ULL; + const auto apply = [&](uint64_t lhs_bits, uint64_t rhs_bits, bool negate_lhs, bool negate_rhs) { + if (negate_lhs) + lhs_bits ^= kSignBit; + if (negate_rhs) + rhs_bits ^= kSignBit; + uint64_t result = amdgpu::fp_mode::binary_f64( + lhs_bits, rhs_bits, amdgpu::fp_mode::BinaryF64Op::MaximumNumber, + wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64()); + return amdgpu::fp_mode::finish_f64(result, wf.fp_round_mode_f16_f64(), 0, inst_.clamp, + amdgpu::floating_clamp_nan_to_zero(wf)); + }; + const uint64_t result_lo = apply(lhs.lo, rhs.lo, (inst_.neg & 1u) != 0, (inst_.neg & 2u) != 0); + const uint64_t result_hi = + apply(lhs.hi, rhs.hi, (inst_.neg_hi & 1u) != 0, (inst_.neg_hi & 2u) != 0); + write_pk_u64_pair(vdst, wf, lane, {result_lo, result_hi}); + } +} + +void VPkMinNumF64Vop3p::execute_impl(amdgpu::Wavefront &wf) { + uint64_t exec = wf.exec(); + for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { + if (!(exec & (1ULL << lane))) + continue; + const auto lhs = read_pk_u64_pair(src0, wf, lane); + const auto rhs = read_pk_u64_pair(src1, wf, lane); + constexpr uint64_t kSignBit = 0x8000000000000000ULL; + const auto apply = [&](uint64_t lhs_bits, uint64_t rhs_bits, bool negate_lhs, bool negate_rhs) { + if (negate_lhs) + lhs_bits ^= kSignBit; + if (negate_rhs) + rhs_bits ^= kSignBit; + uint64_t result = amdgpu::fp_mode::binary_f64( + lhs_bits, rhs_bits, amdgpu::fp_mode::BinaryF64Op::MinimumNumber, + wf.fp_round_mode_f16_f64(), wf.fp_denorm_mode_f16_f64()); + return amdgpu::fp_mode::finish_f64(result, wf.fp_round_mode_f16_f64(), 0, inst_.clamp, + amdgpu::floating_clamp_nan_to_zero(wf)); + }; + const uint64_t result_lo = apply(lhs.lo, rhs.lo, (inst_.neg & 1u) != 0, (inst_.neg & 2u) != 0); + const uint64_t result_hi = + apply(lhs.hi, rhs.hi, (inst_.neg_hi & 1u) != 0, (inst_.neg_hi & 2u) != 0); + write_pk_u64_pair(vdst, wf, lane, {result_lo, result_hi}); + } +} + void VPkLshlAddU64Vop3p::execute_impl(amdgpu::Wavefront &wf) { uint64_t exec = wf.exec(); std::array results{}; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h index 2d4036ed1f68..ea172504f9c1 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h @@ -670,6 +670,56 @@ inline uint64_t fma_f64(uint64_t src0, uint64_t src1, uint64_t src2, uint32_t ro return result; } +/// @brief Binary F64 operations implemented by the shared MODE-aware helper. +enum class BinaryF64Op { Add, Multiply, MaximumNumber, MinimumNumber }; + +/// @brief Execute a binary F64 operation under MODE.FP_ROUND and MODE.FP_DENORM. +/// @details MaximumNumber and MinimumNumber return the numeric operand for a +/// one-NaN input and explicitly select +0/-0 for signed-zero ties respectively. +inline uint64_t binary_f64(uint64_t src0, uint64_t src1, BinaryF64Op operation, uint32_t round_mode, + uint32_t denorm_mode) { + if ((denorm_mode & 1u) == 0) { + src0 = detail::flush_f64(src0); + src1 = detail::flush_f64(src1); + } + + uint64_t result; + { + detail::ScopedFenv environment(round_mode); + const double lhs = std::bit_cast(src0); + const double rhs = std::bit_cast(src1); + const double value = [&] { + switch (operation) { + case BinaryF64Op::Add: + return lhs + rhs; + case BinaryF64Op::Multiply: + return lhs * rhs; + case BinaryF64Op::MaximumNumber: + if (std::isnan(lhs)) + return std::isnan(rhs) ? std::numeric_limits::quiet_NaN() : rhs; + if (std::isnan(rhs)) + return lhs; + if ((src0 & 0x7fffffffffffffffULL) == 0 && (src1 & 0x7fffffffffffffffULL) == 0) + return std::bit_cast((src0 & src1) & 0x8000000000000000ULL); + return std::fmax(lhs, rhs); + case BinaryF64Op::MinimumNumber: + if (std::isnan(lhs)) + return std::isnan(rhs) ? std::numeric_limits::quiet_NaN() : rhs; + if (std::isnan(rhs)) + return lhs; + if ((src0 & 0x7fffffffffffffffULL) == 0 && (src1 & 0x7fffffffffffffffULL) == 0) + return std::bit_cast((src0 | src1) & 0x8000000000000000ULL); + return std::fmin(lhs, rhs); + } + return std::numeric_limits::quiet_NaN(); + }(); + result = std::bit_cast(value); + } + if ((denorm_mode & 2u) == 0) + result = detail::flush_f64(result); + return result; +} + /// @brief Apply F64 OMOD/CLAMP under the architectural rounding mode. /// @details A nonzero OMOD flushes a denormal result and converts either signed zero to +0. inline uint64_t finish_f64(uint64_t value, uint32_t round_mode, uint32_t omod, bool clamp, diff --git a/emulation/rocjitsu/tests/cdna5_execution_test.cpp b/emulation/rocjitsu/tests/cdna5_execution_test.cpp index 0c70c2e1f27c..7048ee2b0bbd 100644 --- a/emulation/rocjitsu/tests/cdna5_execution_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_execution_test.cpp @@ -715,6 +715,8 @@ TEST(FpModePolicyTest, F64HelpersRestoreAmbientHostEnvironment) { for (const uint64_t input : {kOne, kInfinity, kQuietNan}) { (void)amdgpu::fp_mode::fma_f64(input, kOne, kOne, 1, 3); EXPECT_EQ(std::fegetround(), FE_DOWNWARD); + (void)amdgpu::fp_mode::binary_f64(input, kOne, amdgpu::fp_mode::BinaryF64Op::Add, 1, 3); + EXPECT_EQ(std::fegetround(), FE_DOWNWARD); (void)amdgpu::fp_mode::finish_f64(input, 1, 1, false, true); EXPECT_EQ(std::fegetround(), FE_DOWNWARD); } @@ -1774,6 +1776,879 @@ void write_sgpr_packed_u64(amdgpu::ComputeUnitCore &cu, amdgpu::Wavefront &wf, u } } +TEST(Gfx1251PackedF64ExecutionTest, LaterExecutionSlicesRemainCallbackFree) { + // Exact public LLVM encodings for the two instructions intentionally left + // model-only by this slice: gfx1251_asm_vop3p.s and + // gfx1251_asm_wmma_w32.s. + constexpr std::array, 2> kWords{{ + {0xCC3B4004u, 0x1C421908u}, // v_pk_fma_f64 v[4:7], v[8:11], v[12:15], v[16:19] + {0xCC5B0008u, 0x1C220900u}, // v_wmma_f64_16x16x4_f64 v[8:15], v[0:1], v[2:3], v[4:7] + }}; + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + + for (const auto &words : kWords) { + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + EXPECT_EQ(decoded->execute, nullptr) << decoded->mnemonic(); + } +} + +TEST(Gfx1251PackedF64ExecutionTest, BasicArithmeticExecutesBothElementsAndHonorsExec) { + struct Operation { + std::string_view name; + std::array words; + PackedU64Pair expected; + }; + constexpr auto bits = [](double value) { return std::bit_cast(value); }; + constexpr std::array kOperations{ + Operation{"add", {0xCC4B4004u, 0x1A021908u}, {bits(7.0), bits(1.0)}}, + Operation{"mul", {0xCC3C4004u, 0x1A021908u}, {bits(10.0), bits(-12.0)}}, + Operation{"max-num", {0xCC4E4004u, 0x1A021908u}, {bits(5.0), bits(4.0)}}, + Operation{"min-num", {0xCC4F4004u, 0x1A021908u}, {bits(2.0), bits(-3.0)}}, + }; + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(0x5u); + constexpr PackedU64Pair kLhs{bits(2.0), bits(-3.0)}; + constexpr PackedU64Pair kRhs{bits(5.0), bits(4.0)}; + constexpr PackedU64Pair kInactive{0xdeadbeefcafef00dULL, 0xbaadf00d12345678ULL}; + + for (const auto &operation : kOperations) { + SCOPED_TRACE(operation.name); + for (uint32_t lane = 0; lane < 3; ++lane) { + write_vgpr_packed_u64(*cu, *wf, 8, lane, kLhs); + write_vgpr_packed_u64(*cu, *wf, 12, lane, kRhs); + write_vgpr_packed_u64(*cu, *wf, 4, lane, kInactive); + } + std::unique_ptr decoded(decode_valid(*decoder, operation.words.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), operation.expected); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 1), kInactive); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 2), operation.expected); + } +} + +TEST(Gfx1251PackedF64ExecutionTest, ExecutesWave32Lane31WithoutChangingVccOrInactiveLanes) { + constexpr auto bits = [](double value) { return std::bit_cast(value); }; + constexpr std::array kAdd{0xCC4B4004u, 0x1A021908u}; + constexpr PackedU64Pair kInactiveSeed{0x0123456789abcdefULL, 0xfedcba9876543210ULL}; + constexpr uint64_t kVccSeed = 0x5a5aa5a5deadbeefULL; + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + ASSERT_EQ(wf->wf_size(), 32u); + wf->set_exec_raw(uint64_t{1} << 31); + wf->set_vcc_raw(kVccSeed); + write_vgpr_packed_u64(*cu, *wf, 8, 31, {bits(2.0), bits(-3.0)}); + write_vgpr_packed_u64(*cu, *wf, 12, 31, {bits(5.0), bits(4.0)}); + write_vgpr_packed_u64(*cu, *wf, 4, 0, kInactiveSeed); + + std::unique_ptr decoded(decode_valid(*decoder, kAdd.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 31), (PackedU64Pair{bits(7.0), bits(1.0)})); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), kInactiveSeed); + EXPECT_EQ(wf->vcc(), kVccSeed); +} + +TEST(Gfx1251PackedF64ExecutionTest, ExecZeroPreservesDestinationAndStatus) { + constexpr std::array kAdd{0xCC4B4004u, 0x1A021908u}; + constexpr PackedU64Pair kDestinationSeed{0x0123456789abcdefULL, 0xfedcba9876543210ULL}; + constexpr uint64_t kVccSeed = 0x5a5aa5a5deadbeefULL; + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec_raw(0); + wf->set_vcc_raw(kVccSeed); + wf->write_scc(true); + write_vgpr_packed_u64(*cu, *wf, 4, 0, kDestinationSeed); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {0x3ff0000000000000ULL, 0x4000000000000000ULL}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {0x4008000000000000ULL, 0x4010000000000000ULL}); + + std::unique_ptr decoded(decode_valid(*decoder, kAdd.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), kDestinationSeed); + EXPECT_EQ(wf->vcc(), kVccSeed); + EXPECT_TRUE(wf->read_scc()); +} + +TEST(Gfx1251PackedF64ExecutionTest, BasicArithmeticExecutesEveryPublicLlvmSourceForm) { + struct SourceForm { + std::string_view name; + uint32_t second_word; + uint32_t literal; + PackedU64Pair lhs; + PackedU64Pair rhs; + }; + constexpr auto bits = [](double value) { return std::bit_cast(value); }; + constexpr std::array kForms{ + SourceForm{"vgpr-vgpr", 0x1A021908u, 0u, {bits(10.0), bits(20.0)}, {bits(3.0), bits(4.0)}}, + SourceForm{"sgpr-sgpr", 0x1A001808u, 0u, {bits(100.0), bits(100.0)}, {bits(7.0), bits(7.0)}}, + SourceForm{"vgpr-sgpr", 0x1A001908u, 0u, {bits(10.0), bits(20.0)}, {bits(7.0), bits(7.0)}}, + SourceForm{"sgpr-vgpr", 0x1A021808u, 0u, {bits(100.0), bits(100.0)}, {bits(3.0), bits(4.0)}}, + SourceForm{"vgpr-null", 0x1A00F908u, 0u, {bits(10.0), bits(20.0)}, {0u, 0u}}, + SourceForm{ + "vgpr-inline-one", 0x1A01E508u, 0u, {bits(10.0), bits(20.0)}, {bits(1.0), bits(1.0)}}, + SourceForm{ + "inline-one-vgpr", 0x1A0210F2u, 0u, {bits(1.0), bits(1.0)}, {bits(10.0), bits(20.0)}}, + SourceForm{"literal-vgpr", + 0x1A0210FFu, + 0x40594000u, + {bits(101.0), bits(101.0)}, + {bits(10.0), bits(20.0)}}, + SourceForm{"vgpr-literal", + 0x1A01FF08u, + 0x40594000u, + {bits(10.0), bits(20.0)}, + {bits(101.0), bits(101.0)}}, + SourceForm{"shared-literal", + 0x1A01FEFFu, + 0x40594000u, + {bits(101.0), bits(101.0)}, + {bits(101.0), bits(101.0)}}, + }; + struct Operation { + std::string_view name; + uint32_t first_word; + }; + constexpr std::array kOperations{ + Operation{"add", 0xCC4B4004u}, + Operation{"mul", 0xCC3C4004u}, + Operation{"max-num", 0xCC4E4004u}, + Operation{"min-num", 0xCC4F4004u}, + }; + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {bits(10.0), bits(20.0)}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {bits(3.0), bits(4.0)}); + // Packed-FP64 SGPR operands broadcast their low 64 bits. Keep different + // high words in the backing tuple so the result proves they are ignored. + write_sgpr_packed_u64(*cu, *wf, 8, {bits(100.0), bits(200.0)}); + write_sgpr_packed_u64(*cu, *wf, 12, {bits(7.0), bits(8.0)}); + + const auto reference = [](std::string_view operation, double lhs, double rhs) { + if (operation == "add") + return lhs + rhs; + if (operation == "mul") + return lhs * rhs; + if (operation == "max-num") + return std::fmax(lhs, rhs); + return std::fmin(lhs, rhs); + }; + for (const auto &operation : kOperations) { + for (const auto &form : kForms) { + SCOPED_TRACE(operation.name); + SCOPED_TRACE(form.name); + const std::array words{operation.first_word, form.second_word, form.literal}; + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); + const PackedU64Pair expected{ + std::bit_cast(reference(operation.name, std::bit_cast(form.lhs[0]), + std::bit_cast(form.rhs[0]))), + std::bit_cast(reference(operation.name, std::bit_cast(form.lhs[1]), + std::bit_cast(form.rhs[1]))), + }; + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), expected); + if (form.literal != 0u) { + for (uint32_t source = 0; source < 2; ++source) { + if (form.name != "shared-literal" && source != (form.name == "literal-vgpr" ? 0u : 1u)) + continue; + const Operand *literal_operand = decoded->src_operand(source); + ASSERT_NE(literal_operand, nullptr); + EXPECT_EQ(amdgpu::RegisterAccess(*wf).read_lane64(*literal_operand, 0), bits(101.0)); + } + } + } + } +} + +TEST(Gfx1251PackedF64ExecutionTest, ExecutesInlineScalarBoundarySources) { + constexpr auto bits = [](double value) { return std::bit_cast(value); }; + struct InlineCase { + std::string_view name; + uint16_t opcode; + uint16_t src0; + uint16_t src1; + uint64_t expected; + }; + constexpr std::array kCases{ + InlineCase{"positive-integer-low", cdna5::kVPkAddF64Vop3p, 128, 124, 0}, + InlineCase{"positive-integer-high", cdna5::kVPkAddF64Vop3p, 192, 124, 64}, + InlineCase{"negative-integer-low", cdna5::kVPkMaxNumF64Vop3p, 193, 242, bits(1.0)}, + InlineCase{"negative-integer-high", cdna5::kVPkMaxNumF64Vop3p, 208, 242, bits(1.0)}, + InlineCase{"float-half", cdna5::kVPkAddF64Vop3p, 240, 124, bits(0.5)}, + InlineCase{"inverse-two-pi", cdna5::kVPkAddF64Vop3p, 248, 124, 0x3FC45F306DC9C882ULL}, + }; + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + wf->set_mode_raw(3u << 6); + wf->write_scc(true); + + for (const auto &test_case : kCases) { + SCOPED_TRACE(test_case.name); + const auto words = cdna5::build_vop3p(test_case.opcode, {.vdst = 4, + .opsel_hi_2 = 1, + .src0 = test_case.src0, + .src1 = test_case.src1, + .src2 = 128, + .opsel_hi = 3}); + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), + (PackedU64Pair{test_case.expected, test_case.expected})); + } + EXPECT_TRUE(wf->read_scc()); +} + +TEST(Gfx1251PackedF64ExecutionTest, ExecutesFinalRegisterTupleBoundaries) { + constexpr auto bits = [](double value) { return std::bit_cast(value); }; + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + wf->set_mode_raw(3u << 6); + + const auto execute_add = [&](cdna5::Vop3pBuilderFields fields, PackedU64Pair expected) { + const auto words = cdna5::build_vop3p(cdna5::kVPkAddF64Vop3p, fields); + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, fields.vdst, 0), expected); + }; + + write_sgpr_packed_u64(*cu, *wf, 100, {bits(2.0), bits(99.0)}); + execute_add({.vdst = 4, .opsel_hi_2 = 1, .src0 = 100, .src1 = 124, .src2 = 128, .opsel_hi = 3}, + {bits(2.0), bits(2.0)}); + + wf->set_ttmp(12, static_cast(bits(3.0))); + wf->set_ttmp(13, static_cast(bits(3.0) >> 32)); + wf->set_ttmp(14, static_cast(bits(99.0))); + wf->set_ttmp(15, static_cast(bits(99.0) >> 32)); + execute_add({.vdst = 4, .opsel_hi_2 = 1, .src0 = 120, .src1 = 124, .src2 = 128, .opsel_hi = 3}, + {bits(3.0), bits(3.0)}); + + write_vgpr_packed_u64(*cu, *wf, 252, 0, {bits(4.0), bits(5.0)}); + execute_add({.vdst = 252, .opsel_hi_2 = 1, .src0 = 508, .src1 = 100, .src2 = 128, .opsel_hi = 3}, + {bits(6.0), bits(7.0)}); +} + +TEST(Gfx1251PackedF64ExecutionTest, PublicModifiersClampAndOverlappingDestinationAreSafe) { + constexpr auto bits = [](double value) { return std::bit_cast(value); }; + struct Operation { + std::string_view name; + uint32_t base_first_word; + PackedU64Pair neg_src0_expected; + PackedU64Pair neg_src1_expected; + PackedU64Pair clamp_expected; + PackedU64Pair combined_expected; + }; + constexpr std::array kOperations{ + Operation{"add", + 0xCC4B4004u, + {bits(3.0), bits(7.0)}, + {bits(-3.0), bits(-7.0)}, + {bits(0.0), bits(1.0)}, + {bits(1.0), bits(1.0)}}, + Operation{"mul", + 0xCC3C4004u, + {bits(-10.0), bits(12.0)}, + {bits(-10.0), bits(12.0)}, + {bits(0.0), bits(1.0)}, + {bits(0.0), bits(1.0)}}, + Operation{"max-num", + 0xCC4E4004u, + {bits(5.0), bits(4.0)}, + {bits(2.0), bits(-3.0)}, + {bits(1.0), bits(1.0)}, + {bits(1.0), bits(1.0)}}, + Operation{"min-num", + 0xCC4F4004u, + {bits(-2.0), bits(3.0)}, + {bits(-5.0), bits(-4.0)}, + {bits(0.0), bits(1.0)}, + {bits(0.0), bits(1.0)}}, + }; + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + + for (const auto &operation : kOperations) { + SCOPED_TRACE(operation.name); + const std::array modifier_cases{ + std::pair{std::array{operation.base_first_word + 0x100u, 0x3A021908u}, + operation.neg_src0_expected}, + std::pair{std::array{operation.base_first_word + 0x200u, 0x5A021908u}, + operation.neg_src1_expected}, + }; + for (const auto &[words, expected] : modifier_cases) { + write_vgpr_packed_u64(*cu, *wf, 8, 0, {bits(2.0), bits(-3.0)}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {bits(5.0), bits(4.0)}); + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), expected); + } + + const std::array clamp_words{operation.base_first_word + 0x8000u, 0x1A021908u}; + write_vgpr_packed_u64(*cu, *wf, 8, 0, {bits(-2.0), bits(2.0)}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {bits(1.0), bits(2.0)}); + std::unique_ptr clamp(decode_valid(*decoder, clamp_words.data())); + ASSERT_NE(clamp, nullptr); + ASSERT_NE(clamp->execute, nullptr); + EXPECT_TRUE(cu->execute_instruction(clamp.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), operation.clamp_expected); + + const auto combined_words = + cdna5::build_vop3p((operation.base_first_word >> 16) & 0x7fu, {.vdst = 4, + .neg_hi = 1, + .opsel_hi_2 = 1, + .clamp = 1, + .src0 = 264, + .src1 = 268, + .src2 = 128, + .opsel_hi = 3, + .neg = 1}); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {bits(2.0), bits(-3.0)}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {bits(5.0), bits(4.0)}); + std::unique_ptr combined(decode_valid(*decoder, combined_words.data())); + ASSERT_NE(combined, nullptr); + ASSERT_NE(combined->execute, nullptr); + EXPECT_TRUE(cu->execute_instruction(combined.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), operation.combined_expected); + } + + const std::array asymmetric_cases{ + std::pair{cdna5::Vop3pBuilderFields{.vdst = 4, + .opsel_hi_2 = 1, + .src0 = 264, + .src1 = 268, + .src2 = 128, + .opsel_hi = 3, + .neg = 1}, + PackedU64Pair{bits(3.0), bits(1.0)}}, + std::pair{cdna5::Vop3pBuilderFields{.vdst = 4, + .neg_hi = 2, + .opsel_hi_2 = 1, + .src0 = 264, + .src1 = 268, + .src2 = 128, + .opsel_hi = 3}, + PackedU64Pair{bits(7.0), bits(-7.0)}}, + }; + for (const auto &[fields, expected] : asymmetric_cases) { + const auto words = cdna5::build_vop3p(cdna5::kVPkAddF64Vop3p, fields); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {bits(2.0), bits(-3.0)}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {bits(5.0), bits(4.0)}); + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), expected); + } + + constexpr uint64_t kQuietNan = 0x7ff8000000001234ULL; + constexpr uint64_t kSignalingNan = 0x7ff0000000000001ULL; + for (const uint16_t opcode : {cdna5::kVPkAddF64Vop3p, cdna5::kVPkMulF64Vop3p, + cdna5::kVPkMaxNumF64Vop3p, cdna5::kVPkMinNumF64Vop3p}) { + const auto clamp_nan_words = cdna5::build_vop3p(opcode, {.vdst = 4, + .opsel_hi_2 = 1, + .clamp = 1, + .src0 = 264, + .src1 = 124, + .src2 = 128, + .opsel_hi = 3}); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {kQuietNan, kSignalingNan}); + std::unique_ptr clamp_nan(decode_valid(*decoder, clamp_nan_words.data())); + ASSERT_NE(clamp_nan, nullptr); + ASSERT_NE(clamp_nan->execute, nullptr); + EXPECT_TRUE(cu->execute_instruction(clamp_nan.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), (PackedU64Pair{bits(0.0), bits(0.0)})); + } + + struct OverlapCase { + std::string_view name; + uint8_t vdst; + }; + constexpr std::array kOverlapCases{ + OverlapCase{"exact-src0", 8}, + OverlapCase{"exact-src1", 12}, + OverlapCase{"partial-both", 10}, + }; + for (const auto &test_case : kOverlapCases) { + SCOPED_TRACE(test_case.name); + const auto overlap_words = cdna5::build_vop3p(cdna5::kVPkAddF64Vop3p, {.vdst = test_case.vdst, + .opsel_hi_2 = 1, + .src0 = 264, + .src1 = 268, + .src2 = 128, + .opsel_hi = 3}); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {bits(2.0), bits(-3.0)}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {bits(5.0), bits(4.0)}); + std::unique_ptr overlap(decode_valid(*decoder, overlap_words.data())); + ASSERT_NE(overlap, nullptr); + ASSERT_NE(overlap->execute, nullptr); + EXPECT_TRUE(cu->execute_instruction(overlap.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, test_case.vdst, 0), + (PackedU64Pair{bits(7.0), bits(1.0)})); + } +} + +TEST(Gfx1251PackedF64ExecutionTest, AddAndMultiplyHonorRoundingAndDenormModes) { + constexpr auto bits = [](double value) { return std::bit_cast(value); }; + const auto add_words = cdna5::build_vop3p( + cdna5::kVPkAddF64Vop3p, + {.vdst = 4, .opsel_hi_2 = 1, .src0 = 264, .src1 = 268, .src2 = 128, .opsel_hi = 3}); + const auto mul_words = cdna5::build_vop3p( + cdna5::kVPkMulF64Vop3p, + {.vdst = 4, .opsel_hi_2 = 1, .src0 = 264, .src1 = 268, .src2 = 128, .opsel_hi = 3}); + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + std::unique_ptr add(decode_valid(*decoder, add_words.data())); + std::unique_ptr mul(decode_valid(*decoder, mul_words.data())); + ASSERT_NE(add, nullptr); + ASSERT_NE(mul, nullptr); + ASSERT_NE(add->execute, nullptr); + ASSERT_NE(mul->execute, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + + constexpr uint64_t kOne = bits(1.0); + constexpr uint64_t kNegativeOne = bits(-1.0); + constexpr uint64_t kHalfUlpAtOne = bits(0x1p-53); + constexpr uint64_t kNegativeHalfUlpAtOne = bits(-0x1p-53); + constexpr std::array kAddExpected{{ + {kOne, kNegativeOne}, + {kOne + 1, kNegativeOne}, + {kOne, kNegativeOne + 1}, + {kOne, kNegativeOne}, + }}; + for (uint32_t round = 0; round < 4; ++round) { + wf->set_mode_raw((round << 2) | (3u << 6)); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {kOne, kNegativeOne}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {kHalfUlpAtOne, kNegativeHalfUlpAtOne}); + EXPECT_TRUE(cu->execute_instruction(add.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), kAddExpected[round]) << round; + } + + constexpr uint64_t kNextOne = kOne + 1; + constexpr uint64_t kRoundedSquare = kOne + 2; + constexpr std::array kMulExpected{kRoundedSquare, kRoundedSquare + 1, kRoundedSquare, + kRoundedSquare}; + for (uint32_t round = 0; round < 4; ++round) { + wf->set_mode_raw((round << 2) | (3u << 6)); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {kNextOne, kNextOne}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {kNextOne, kNextOne}); + EXPECT_TRUE(cu->execute_instruction(mul.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), + (PackedU64Pair{kMulExpected[round], kMulExpected[round]})) + << round; + } + + constexpr uint64_t kMinSubnormal = 1u; + constexpr uint64_t kHalfMinNormal = 0x0008000000000000ULL; + constexpr uint64_t kMinNormal = 0x0010000000000000ULL; + constexpr std::array kAddDenormExpected{0u, 0u, 0u, kMinSubnormal}; + constexpr std::array kMulDenormExpected{0u, 0u, kHalfMinNormal, kHalfMinNormal}; + for (uint32_t denorm = 0; denorm < 4; ++denorm) { + wf->set_mode_raw(denorm << 6); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {kMinSubnormal, kMinNormal}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {0u, bits(0.5)}); + EXPECT_TRUE(cu->execute_instruction(add.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0)[0], kAddDenormExpected[denorm]) << denorm; + EXPECT_TRUE(cu->execute_instruction(mul.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0)[1], kMulDenormExpected[denorm]) << denorm; + } +} + +TEST(Gfx1251PackedF64ExecutionTest, MinMaxNumberHonorEveryDenormMode) { + constexpr uint64_t kPositiveZero = 0; + constexpr uint64_t kNegativeZero = 0x8000000000000000ULL; + constexpr uint64_t kMinSubnormal = 1; + constexpr uint64_t kNegativeMinSubnormal = kNegativeZero | kMinSubnormal; + constexpr std::array kMaxExpected{kPositiveZero, kPositiveZero, kPositiveZero, + kMinSubnormal}; + constexpr std::array kMinExpected{kNegativeZero, kNegativeZero, kNegativeZero, + kNegativeMinSubnormal}; + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + const auto max_words = cdna5::build_vop3p( + cdna5::kVPkMaxNumF64Vop3p, + {.vdst = 4, .opsel_hi_2 = 1, .src0 = 264, .src1 = 268, .src2 = 128, .opsel_hi = 3}); + const auto min_words = cdna5::build_vop3p( + cdna5::kVPkMinNumF64Vop3p, + {.vdst = 4, .opsel_hi_2 = 1, .src0 = 264, .src1 = 268, .src2 = 128, .opsel_hi = 3}); + std::unique_ptr maximum(decode_valid(*decoder, max_words.data())); + std::unique_ptr minimum(decode_valid(*decoder, min_words.data())); + ASSERT_NE(maximum, nullptr); + ASSERT_NE(minimum, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + + for (uint32_t denorm = 0; denorm < 4; ++denorm) { + wf->set_mode_raw(denorm << 6); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {kMinSubnormal, kNegativeMinSubnormal}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {kPositiveZero, kPositiveZero}); + EXPECT_TRUE(cu->execute_instruction(maximum.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0)[0], kMaxExpected[denorm]) << denorm; + EXPECT_TRUE(cu->execute_instruction(minimum.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0)[1], kMinExpected[denorm]) << denorm; + } +} + +TEST(Gfx1251PackedF64ExecutionTest, AddAndMultiplyHandleNanInfinityAndSignedZero) { + constexpr auto bits = [](double value) { return std::bit_cast(value); }; + constexpr uint64_t kPositiveZero = 0u; + constexpr uint64_t kNegativeZero = 0x8000000000000000ULL; + constexpr uint64_t kPositiveInfinity = bits(std::numeric_limits::infinity()); + constexpr uint64_t kNegativeInfinity = bits(-std::numeric_limits::infinity()); + constexpr std::array, 2> kCases{{ + {cdna5::kVPkAddF64Vop3p, {kNegativeInfinity, kNegativeZero}}, + {cdna5::kVPkMulF64Vop3p, {kPositiveZero, bits(2.0)}}, + }}; + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + wf->set_mode_raw(3u << 6); + + for (const auto &[opcode, rhs] : kCases) { + const auto words = cdna5::build_vop3p( + opcode, {.vdst = 4, .opsel_hi_2 = 1, .src0 = 264, .src1 = 268, .src2 = 128, .opsel_hi = 3}); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {kPositiveInfinity, kNegativeZero}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, rhs); + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); + const PackedU64Pair result = read_vgpr_packed_u64(*cu, *wf, 4, 0); + EXPECT_TRUE(std::isnan(std::bit_cast(result[0]))); + EXPECT_EQ(result[1], kNegativeZero); + } +} + +TEST(Gfx1251PackedF64ExecutionTest, MinMaxNumberHandleNanInfinityAndSignedZero) { + constexpr auto bits = [](double value) { return std::bit_cast(value); }; + constexpr uint64_t kQuietNan = 0x7ff8000000001234ULL; + constexpr uint64_t kSignalingNan = 0x7ff0000000000001ULL; + constexpr uint64_t kPositiveZero = 0u; + constexpr uint64_t kNegativeZero = 0x8000000000000000ULL; + constexpr uint64_t kPositiveInfinity = bits(std::numeric_limits::infinity()); + constexpr uint64_t kNegativeInfinity = bits(-std::numeric_limits::infinity()); + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + wf->set_mode_raw(3u << 6); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {kQuietNan, kNegativeZero}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {bits(2.0), kPositiveZero}); + + for (const auto &[words, expected] : {std::pair{std::array{0xCC4E4004u, 0x1A021908u}, + PackedU64Pair{bits(2.0), kPositiveZero}}, + std::pair{std::array{0xCC4F4004u, 0x1A021908u}, + PackedU64Pair{bits(2.0), kNegativeZero}}}) { + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), expected); + } + + write_vgpr_packed_u64(*cu, *wf, 8, 0, {kQuietNan, bits(3.0)}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {bits(2.0), kSignalingNan}); + for (const auto &words : {std::array{0xCC4E4004u, 0x1A021908u}, + std::array{0xCC4F4004u, 0x1A021908u}}) { + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), (PackedU64Pair{bits(2.0), bits(3.0)})); + } + + struct NanCase { + std::string_view name; + uint64_t lhs; + uint64_t rhs; + uint64_t expected; + bool expect_nan; + }; + constexpr std::array kNanCases{ + NanCase{"qnan-left", kQuietNan, bits(2.0), bits(2.0), false}, + NanCase{"qnan-right", bits(2.0), kQuietNan, bits(2.0), false}, + NanCase{"snan-left", kSignalingNan, bits(3.0), bits(3.0), false}, + NanCase{"snan-right", bits(3.0), kSignalingNan, bits(3.0), false}, + NanCase{"qnan-snan", kQuietNan, kSignalingNan, 0, true}, + NanCase{"snan-qnan", kSignalingNan, kQuietNan, 0, true}, + }; + for (const auto &words : {std::array{0xCC4E4004u, 0x1A021908u}, + std::array{0xCC4F4004u, 0x1A021908u}}) { + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + for (const auto &test_case : kNanCases) { + SCOPED_TRACE(decoded->mnemonic()); + SCOPED_TRACE(test_case.name); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {test_case.lhs, test_case.lhs}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {test_case.rhs, test_case.rhs}); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); + const PackedU64Pair result = read_vgpr_packed_u64(*cu, *wf, 4, 0); + if (test_case.expect_nan) { + EXPECT_TRUE(std::isnan(std::bit_cast(result[0]))); + EXPECT_TRUE(std::isnan(std::bit_cast(result[1]))); + } else { + EXPECT_EQ(result, (PackedU64Pair{test_case.expected, test_case.expected})); + } + } + } + + write_vgpr_packed_u64(*cu, *wf, 8, 0, {kPositiveZero, kNegativeZero}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {kNegativeZero, kPositiveZero}); + for (const auto &[words, expected] : {std::pair{std::array{0xCC4E4004u, 0x1A021908u}, + PackedU64Pair{kPositiveZero, kPositiveZero}}, + std::pair{std::array{0xCC4F4004u, 0x1A021908u}, + PackedU64Pair{kNegativeZero, kNegativeZero}}}) { + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), expected); + } + + write_vgpr_packed_u64(*cu, *wf, 8, 0, {kNegativeInfinity, kQuietNan}); + write_vgpr_packed_u64(*cu, *wf, 12, 0, {kPositiveInfinity, kQuietNan}); + for (const auto &words : {std::array{0xCC4E4004u, 0x1A021908u}, + std::array{0xCC4F4004u, 0x1A021908u}}) { + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); + const PackedU64Pair result = read_vgpr_packed_u64(*cu, *wf, 4, 0); + EXPECT_EQ(result[0], decoded->mnemonic() == std::string_view("v_pk_max_num_f64") + ? kPositiveInfinity + : kNegativeInfinity); + EXPECT_TRUE(std::isnan(std::bit_cast(result[1]))); + } +} + +TEST(Gfx1251PackedF64ExecutionTest, ExecutesSgpr104CompositeInBothSourcesForEveryOperation) { + constexpr auto bits = [](double value) { return std::bit_cast(value); }; + struct Operation { + std::string_view name; + uint32_t first_word; + PackedU64Pair expected; + }; + constexpr std::array kOperations{ + Operation{"add", 0xCC4B4004u, {bits(5.0), bits(6.0)}}, + Operation{"mul", 0xCC3C4004u, {bits(6.0), bits(8.0)}}, + Operation{"max-num", 0xCC4E4004u, {bits(3.0), bits(4.0)}}, + Operation{"min-num", 0xCC4F4004u, {bits(2.0), bits(2.0)}}, + }; + // Exact encodings produced by public LLVM MC for the SGPR104_128 composite + // ([s104,s105,vcc_lo,vcc_hi]) in both VSrc_v2f64 positions. + constexpr uint32_t kSgpr104Src0 = 0x1A021068u; + constexpr uint32_t kSgpr104Src1 = 0x1A00D108u; + + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + + constexpr uint64_t kVccPoison = 0xdeadbeefcafef00dULL; + write_wave_sgpr(*cu, *wf, 104, static_cast(bits(2.0))); + write_wave_sgpr(*cu, *wf, 105, static_cast(bits(2.0) >> 32)); + wf->set_vcc_raw(kVccPoison); + write_vgpr_packed_u64(*cu, *wf, 8, 0, {bits(3.0), bits(4.0)}); + + for (const auto &operation : kOperations) { + for (const bool use_src1 : {false, true}) { + SCOPED_TRACE(operation.name); + SCOPED_TRACE(use_src1 ? "src1" : "src0"); + const std::array words{ + operation.first_word, + use_src1 ? kSgpr104Src1 : kSgpr104Src0, + }; + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + ASSERT_NE(decoded, nullptr); + ASSERT_NE(decoded->execute, nullptr); + EXPECT_TRUE(cu->execute_instruction(decoded.get(), *wf).succeeded()); + EXPECT_EQ(read_vgpr_packed_u64(*cu, *wf, 4, 0), operation.expected); + EXPECT_EQ(wf->vcc(), kVccPoison); + } + } +} + +TEST(Gfx1251PackedF64ExecutionTest, RejectsUndefinedLayoutsAndOutOfRangeRegisterTuples) { + auto decoder = + make_isa_decoder(&cdna5::execution_backend(), cdna5::kGfx1251IsaFeatures); + ASSERT_NE(decoder, nullptr); + constexpr std::array kOpcodes{ + cdna5::kVPkAddF64Vop3p, + cdna5::kVPkMulF64Vop3p, + cdna5::kVPkMaxNumF64Vop3p, + cdna5::kVPkMinNumF64Vop3p, + }; + constexpr std::array kInvalidFields{ + cdna5::Vop3pBuilderFields{.vdst = 4, + .opsel = 1, + .opsel_hi_2 = 1, + .src0 = 264, + .src1 = 268, + .src2 = 128, + .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{ + .vdst = 4, .opsel_hi_2 = 0, .src0 = 264, .src1 = 268, .src2 = 128, .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{ + .vdst = 4, .opsel_hi_2 = 1, .src0 = 264, .src1 = 268, .src2 = 129, .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{ + .vdst = 253, .opsel_hi_2 = 1, .src0 = 264, .src1 = 268, .src2 = 128, .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{ + .vdst = 4, .opsel_hi_2 = 1, .src0 = 509, .src1 = 268, .src2 = 128, .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{.vdst = 4, + .neg_hi = 4, + .opsel_hi_2 = 1, + .src0 = 264, + .src1 = 268, + .src2 = 128, + .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{.vdst = 4, + .opsel_hi_2 = 1, + .src0 = 264, + .src1 = 268, + .src2 = 128, + .opsel_hi = 3, + .neg = 4}, + cdna5::Vop3pBuilderFields{ + .vdst = 3, .opsel_hi_2 = 1, .src0 = 264, .src1 = 268, .src2 = 128, .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{ + .vdst = 4, .opsel_hi_2 = 1, .src0 = 265, .src1 = 268, .src2 = 128, .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{ + .vdst = 4, .opsel_hi_2 = 1, .src0 = 10, .src1 = 268, .src2 = 128, .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{ + .vdst = 4, .opsel_hi_2 = 1, .src0 = 109, .src1 = 268, .src2 = 128, .opsel_hi = 3}, + }; + for (const uint16_t opcode : kOpcodes) { + SCOPED_TRACE(opcode); + for (const auto &fields : kInvalidFields) { + const auto words = cdna5::build_vop3p(opcode, fields); + EXPECT_EQ(decode_valid(*decoder, words.data()), nullptr); + } + } + + constexpr std::array kInvalidV2F64Selectors{ + 106, 107, 125, 126, 127, 209, 229, 230, 231, 232, 234, + 235, 236, 237, 238, 239, 249, 251, 252, 253, 254, + }; + for (const uint16_t opcode : kOpcodes) { + SCOPED_TRACE(opcode); + for (const uint16_t selector : kInvalidV2F64Selectors) { + for (const bool use_src1 : {false, true}) { + SCOPED_TRACE(selector); + SCOPED_TRACE(use_src1); + const auto words = cdna5::build_vop3p(opcode, {.vdst = 4, + .opsel_hi_2 = 1, + .src0 = use_src1 ? uint16_t{264} : selector, + .src1 = use_src1 ? selector : uint16_t{268}, + .src2 = 128, + .opsel_hi = 3}); + EXPECT_EQ(decode_valid(*decoder, words.data()), nullptr); + } + } + } + + constexpr std::array kValidV2F64Selectors{ + 104, 124, 128, 208, 240, 248, 255, + }; + for (const uint16_t opcode : kOpcodes) { + SCOPED_TRACE(opcode); + for (const uint16_t selector : kValidV2F64Selectors) { + SCOPED_TRACE(selector); + const auto encoding = cdna5::build_vop3p( + opcode, + {.vdst = 4, .opsel_hi_2 = 1, .src0 = selector, .src1 = 268, .src2 = 128, .opsel_hi = 3}); + const std::array words{encoding[0], encoding[1], 0x65u}; + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + EXPECT_NE(decoded, nullptr); + } + } + + constexpr std::array kValidBoundaries{ + cdna5::Vop3pBuilderFields{ + .vdst = 252, .opsel_hi_2 = 1, .src0 = 508, .src1 = 100, .src2 = 128, .opsel_hi = 3}, + cdna5::Vop3pBuilderFields{ + .vdst = 2, .opsel_hi_2 = 1, .src0 = 258, .src1 = 262, .src2 = 128, .opsel_hi = 3}, + }; + for (const uint16_t opcode : kOpcodes) { + SCOPED_TRACE(opcode); + for (const auto &fields : kValidBoundaries) { + const auto words = cdna5::build_vop3p(opcode, fields); + std::unique_ptr decoded(decode_valid(*decoder, words.data())); + EXPECT_NE(decoded, nullptr); + } + } +} + TEST(Gfx1251PackedU64ExecutionTest, AddAndSubtractHaveExecutionCallbacks) { // Exact public LLVM gfx1251_asm_vop3p.s encodings for the VGPR forms. constexpr std::array, 2> kWords{{ diff --git a/emulation/rocjitsu/tests/dbi/dbi_arg_sim_test.cpp b/emulation/rocjitsu/tests/dbi/dbi_arg_sim_test.cpp index 71360e12190c..3d5a0c0b4990 100644 --- a/emulation/rocjitsu/tests/dbi/dbi_arg_sim_test.cpp +++ b/emulation/rocjitsu/tests/dbi/dbi_arg_sim_test.cpp @@ -109,7 +109,7 @@ class DbiArgSimBase : public ::testing::Test { pt.anchor_offset = 4; // v_mov_b32 v1, v0 -> reads v0 (v0 live at the anchor). pt.probe_obj = &probe_obj; pt.probe_symbol = "rj_test_arg_probe"; - pt.probe_args = {kArgSentinel}; + pt.probe_args.push_back(kArgSentinel); instr.add_point(pt); auto result = instr.patch_with_debug_summaries(); diff --git a/shared/machine-readable-isa/isa/amdgpu_isa_cdna5_variants.json b/shared/machine-readable-isa/isa/amdgpu_isa_cdna5_variants.json index b9a63c93bfdb..d93b357026e6 100644 --- a/shared/machine-readable-isa/isa/amdgpu_isa_cdna5_variants.json +++ b/shared/machine-readable-isa/isa/amdgpu_isa_cdna5_variants.json @@ -46,10 +46,6 @@ ], "model_only_instructions": [ "V_PK_FMA_F64", - "V_PK_MUL_F64", - "V_PK_ADD_F64", - "V_PK_MAX_NUM_F64", - "V_PK_MIN_NUM_F64", "V_WMMA_F64_16X16X4_F64" ], "encodings": [