hanhanW · Sep 2, 2026 · Aug 24, 2026 · Aug 27, 2026 · Sep 1, 2026
@@ -5296,6 +5296,54 @@ def _emit_cdna5_matrix_fmt_helpers(self) -> _ImplOutputs:
               const auto pair = amdgpu::RegisterAccess(wf).read_lane_pair32(operand, lane);
               return {pair.lo, pair.hi};
             }
+
+            struct PkU64Pair {
+              uint64_t lo;
+              uint64_t hi;
+            };
+
+            struct PkU32Pair {
+              uint32_t lo;
+              uint32_t hi;
+            };
+
+            Operand packed_register_dword_offset(const Operand &operand, uint32_t dword_offset) {
+              Operand shifted = operand;
+              shifted.encoding_value_ += static_cast<int>(dword_offset);
+              return shifted;
+            }
+
+            PkU64Pair read_pk_u64_pair(const Operand &operand, const amdgpu::Wavefront &wf,
+                                       uint32_t lane) {
+              const uint64_t lo = amdgpu::RegisterAccess(wf).read_lane64(operand, lane);
+              const auto reg = operand.to_register_ref();
+              if (!reg || reg->cls != RegClass::VGPR)
+                return {lo, lo};
+
+              const Operand hi_operand = packed_register_dword_offset(operand, 2);
+              return {lo, amdgpu::RegisterAccess(wf).read_lane64(hi_operand, lane)};
+            }
+
+            PkU32Pair read_pk_u32_pair(const Operand &operand, const amdgpu::Wavefront &wf,
+                                       uint32_t lane) {
+              // GFX12+ single-SGPR-read operands read the first SGPR and replicate it.
+              // VGPRs and 64-bit special registers such as VCC and EXEC remain pairs.
+              const auto reg = operand.to_register_ref();
+              if (reg && reg->cls == RegClass::SGPR) {
+                const uint32_t value = amdgpu::RegisterAccess(wf).read_lane(operand, lane);
+                return {value, value};
+              }
+              const auto pair = amdgpu::RegisterAccess(wf).read_lane_pair32(operand, lane);
+              return {pair.lo, pair.hi};
+            }
+
+            void write_pk_u64_pair(const Operand &operand, amdgpu::Wavefront &wf, uint32_t lane,
+                                   PkU64Pair value) {
+              const Operand hi_operand = packed_register_dword_offset(operand, 2);
+              amdgpu::RegisterAccess access(wf);
+              access.write_lane64(operand, lane, value.lo);
+              access.write_lane64(hi_operand, lane, value.hi);
+            }
             ''')
         model = (
             'namespace {\n\n'

@@ -119,6 +119,7 @@ def _register_handlers() -> None:
         gen_pk_fmac_vop3,
         gen_pk_binop_f32,
         gen_pk_ternary_f32,
+        gen_pk_lshl_add_u64,
         gen_pk_mov_b32,
         gen_mad_mix_f32,
         gen_mad_mix_lo_hi,
@@ -329,6 +330,7 @@ def _register_handlers() -> None:
         opsel_exprs=c.opsel_exprs,
         use_cdna5_helpers=c.arch_name == 'cdna5',
     )
+    DISPATCH['pk_lshl_add_u64'] = lambda c: gen_pk_lshl_add_u64(c.dst_ops, c.src_ops)
     DISPATCH['pk_mov_b32'] = lambda c: gen_pk_mov_b32(
         c.dst_ops,
         c.src_ops,

@@ -638,6 +638,44 @@ def gen_pk_ternary_f32(
     return '\n'.join(L)
 
 
+def gen_pk_lshl_add_u64(dst: list[str], src: list[str]) -> str:
+    """Generate packed U64 shift-left-add over two independent elements.
+
+    V_PK_LSHL_ADD_U64 has two 64-bit values in each 128-bit value/addend
+    operand and two 32-bit shift counts in its 64-bit shift operand. VGPR U64
+    sources supply both elements, while scalar-backed U64 sources broadcast the
+    low 64 bits. Its public LLVM profile explicitly disables clamp and source
+    modifiers, so the body intentionally consumes neither field. The currently
+    proven shift-count range is 0..4; execution fails closed before any
+    destination write if either count of any active lane is outside that range.
+    """
+    d, s0, s1, s2 = dst[0], src[0], src[1], src[2]
+    return '\n'.join(
+        [
+            '  uint64_t exec = wf.exec();',
+            '  std::array<PkU64Pair, 64> results{};',
+            '  for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) {',
+            '    if (!(exec & (1ULL << lane))) continue;',
+            f'    const auto values = read_pk_u64_pair({s0}, wf, lane);',
+            f'    const auto shifts = read_pk_u32_pair({s1}, wf, lane);',
+            f'    const auto addends = read_pk_u64_pair({s2}, wf, lane);',
+            '    if (shifts.lo > 4u || shifts.hi > 4u) {',
+            '      wf.report_instruction_execution_error(',
+            '          amdgpu::InstructionExecutionError::UnsupportedOperandValue);',
+            '      return;',
+            '    }',
+            '    const uint64_t result_lo = amdgpu::lshl_masked(values.lo, static_cast<uint64_t>(shifts.lo)) + addends.lo;',
+            '    const uint64_t result_hi = amdgpu::lshl_masked(values.hi, static_cast<uint64_t>(shifts.hi)) + addends.hi;',
+            '    results[lane] = {result_lo, result_hi};',
+            '  }',
+            '  for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) {',
+            '    if (!(exec & (1ULL << lane))) continue;',
+            f'    write_pk_u64_pair({d}, wf, lane, results[lane]);',
+            '  }',
+        ]
+    )
+
+
 def gen_pk_mov_b32(
     dst: list[str],
     src: list[str],

@@ -1603,6 +1603,10 @@ def _derive_vop3p(name: str) -> InstructionSemantics | None:
         return InstructionSemantics(
             name, 'pk_binop_f32', operation='add', data_type='f32'
         )
+    if name == 'V_PK_LSHL_ADD_U64':
+        return InstructionSemantics(
+            name, 'pk_lshl_add_u64', operation='lshl_add', data_type='u64'
+        )
     if name == 'V_PK_MOV_B32':
         return InstructionSemantics(name, 'pk_mov_b32')
 

@@ -5083,7 +5083,6 @@ def test_cdna5_model_only_variant_instructions_have_no_execution_callbacks(
         'VPkSubNcU64Vop3p',
         'VPkMaxNumF64Vop3p',
         'VPkMinNumF64Vop3p',
-        'VPkLshlAddU64Vop3p',
         'VWmmaF6416x16x4F64Vop3p',
     )
     header = (gfx1250_generated_root / 'vop3p.h').read_text()
@@ -5103,6 +5102,32 @@ def test_cdna5_model_only_variant_instructions_have_no_execution_callbacks(
         assert class_name not in backend_source
         assert class_name not in execution_source
 
+    executable_class = 'VPkLshlAddU64Vop3p'
+    class_body = header.split(f'class {executable_class} ', 1)[1].split('\n};', 1)[0]
+    assert 'execute_impl' in class_body
+    constructor = model.split(f'{executable_class}::{executable_class}(', 1)[1].split(
+        '\n}', 1
+    )[0]
+    assert 'selected_exec_fn(InstructionExecutionId::VPkLshlAddU64Vop3p)' in constructor
+    assert executable_class in backend_header
+    assert executable_class in backend_source
+    assert executable_class in execution_source
+    assert 'PkU64Pair read_pk_u64_pair' in execution_source
+    assert 'PkU32Pair read_pk_u32_pair' in execution_source
+    assert 'void write_pk_u64_pair' in execution_source
+    u64_read_helper = execution_source.split('PkU64Pair read_pk_u64_pair', 1)[1].split(
+        '\n}', 1
+    )[0]
+    assert 'const auto reg = operand.to_register_ref();' in u64_read_helper
+    assert 'if (!reg || reg->cls != RegClass::VGPR)' in u64_read_helper
+    u32_read_helper = execution_source.split('PkU32Pair read_pk_u32_pair', 1)[1].split(
+        '\n}', 1
+    )[0]
+    assert 'const auto reg = operand.to_register_ref();' in u32_read_helper
+    assert 'if (reg && reg->cls == RegClass::SGPR)' in u32_read_helper
+    assert 'read_lane(operand, lane)' in u32_read_helper
+    assert 'read_lane_pair32(operand, lane)' in u32_read_helper
+
 
 def test_generated_vop_execution_has_no_instruction_storage_bypass(
     amdgpu_generated_root: Path,

@@ -15,6 +15,7 @@
     gen_pk_binop_f32,
     gen_pk_fmac_vop2,
     gen_pk_fmac_vop3,
+    gen_pk_lshl_add_u64,
     gen_pk_ternary,
 )
 from amdisa.codegen.execute.simd_codegen import vop3p_local_simd_probe_line
@@ -289,6 +290,40 @@ def test_cdna_pk_f32_reads_all_register_pairs():
     assert 'encoding_value_ >= 256' not in cpp
 
 
+def test_pk_lshl_add_u64_operates_on_two_independent_64_bit_elements():
+    cpp = gen_pk_lshl_add_u64(['vdst'], ['src0', 'src1', 'src2'])
+
+    assert 'const auto values = read_pk_u64_pair(src0, wf, lane);' in cpp
+    assert 'const auto shifts = read_pk_u32_pair(src1, wf, lane);' in cpp
+    assert 'const auto addends = read_pk_u64_pair(src2, wf, lane);' in cpp
+    assert (
+        'amdgpu::lshl_masked(values.lo, static_cast<uint64_t>(shifts.lo)) + addends.lo'
+        in cpp
+    )
+    assert (
+        'amdgpu::lshl_masked(values.hi, static_cast<uint64_t>(shifts.hi)) + addends.hi'
+        in cpp
+    )
+    assert 'lshl_masked(values.lo, shifts.lo)' not in cpp
+    assert 'lshl_masked(values.hi, shifts.hi)' not in cpp
+    assert 'results[lane] = {result_lo, result_hi};' in cpp
+    assert 'write_pk_u64_pair(vdst, wf, lane, results[lane]);' in cpp
+    assert 'inst_.neg' not in cpp
+    assert 'inst_.clamp' not in cpp
+
+
+def test_pk_lshl_add_u64_rejects_unproven_counts_before_any_write():
+    cpp = gen_pk_lshl_add_u64(['vdst'], ['src0', 'src1', 'src2'])
+
+    reject = 'if (shifts.lo > 4u || shifts.hi > 4u)'
+    report = 'wf.report_instruction_execution_error('
+    write = 'write_pk_u64_pair(vdst, wf, lane, results[lane]);'
+    assert reject in cpp
+    assert report in cpp
+    assert write in cpp
+    assert cpp.index(reject) < cpp.index(report) < cpp.index(write)
+
+
 def test_renamed_vop3p_packed_f32_probe_passes_profile_selectors():
     probe = vop3p_local_simd_probe_line('v_pk_add_f32_vop3p', ('opsel', 'opsel_hi'))
 

@@ -2493,6 +2493,14 @@ def test_pk_binop_f32(self):
         block = derive_sema_block(sem)
         assert block is not None
 
+    def test_pk_lshl_add_u64_has_distinct_packed_width_semantics(self):
+        sem = derive_semantics('V_PK_LSHL_ADD_U64', 'ENC_VOP3P')
+
+        assert sem is not None
+        assert sem.semantic_class == 'pk_lshl_add_u64'
+        assert sem.operation == 'lshl_add'
+        assert sem.data_type == 'u64'
+
     def test_pk_mov_b32(self):
         sem = _FakeSem('V_PK_MOV_B32', 'pk_mov_b32')
         block = derive_sema_block(sem)

@@ -1269,6 +1269,7 @@ enum class InstructionExecutionId : size_t {
   VWmmaF1616x16x128Bf8Fp8Vop3p,
   VWmmaF1616x16x128Bf8Bf8Vop3p,
   VWmmaF3232x16x128F4Vop3p,
+  VPkLshlAddU64Vop3p,
   VWmmaScaleF32Vop3px2,
   DsAddU32Vds,
   DsSubU32Vds,

@@ -1276,6 +1276,7 @@ constexpr InstructionCallbackTable kInstructionCallbacks{{
     &execute_with_backend<VWmmaF1616x16x128Bf8Fp8Vop3p>,
     &execute_with_backend<VWmmaF1616x16x128Bf8Bf8Vop3p>,
     &execute_with_backend<VWmmaF3232x16x128F4Vop3p>,
+    &execute_with_backend<VPkLshlAddU64Vop3p>,
     &execute_with_backend<VWmmaScaleF32Vop3px2>,
     &execute_with_backend<DsAddU32Vds>,
     &execute_with_backend<DsSubU32Vds>,

@@ -4461,7 +4461,8 @@ DecodeResult decodeVPkMinNumF64Vop3p(const MachineInst *opcode,
 } // namespace detail
 
 VPkLshlAddU64Vop3p::VPkLshlAddU64Vop3p(const MachineInst *inst)
-    : Vop3p("v_pk_lshl_add_u64", reinterpret_cast<const OpEncoding *>(inst), nullptr),
+    : Vop3p("v_pk_lshl_add_u64", reinterpret_cast<const OpEncoding *>(inst),
+            selected_exec_fn(InstructionExecutionId::VPkLshlAddU64Vop3p)),
       vdst(128, OperandType::OPR_VGPR, reinterpret_cast<const OpEncoding *>(inst)->vdst),
       src0(128, OperandType::OPR_SRC, reinterpret_cast<const OpEncoding *>(inst)->src0),
       src1(64, OperandType::OPR_SRC, reinterpret_cast<const OpEncoding *>(inst)->src1),

@@ -992,6 +992,7 @@ class VPkMinNumF64Vop3p : public Vop3p {
 class VPkLshlAddU64Vop3p : public Vop3p {
 public:
   VPkLshlAddU64Vop3p(const MachineInst *inst);
+  void execute_impl(amdgpu::Wavefront &wf);
   Operand vdst;
   Operand src0;
   Operand src1;

@@ -36,6 +36,52 @@ PkF32Words read_pk_f32_words(const Operand &operand, const amdgpu::Wavefront &wf
   return {pair.lo, pair.hi};
 }
 
+struct PkU64Pair {
+  uint64_t lo;
+  uint64_t hi;
+};
+
+struct PkU32Pair {
+  uint32_t lo;
+  uint32_t hi;
+};
+
+Operand packed_register_dword_offset(const Operand &operand, uint32_t dword_offset) {
+  Operand shifted = operand;
+  shifted.encoding_value_ += static_cast<int>(dword_offset);
+  return shifted;
+}
+
+PkU64Pair read_pk_u64_pair(const Operand &operand, const amdgpu::Wavefront &wf, uint32_t lane) {
+  const uint64_t lo = amdgpu::RegisterAccess(wf).read_lane64(operand, lane);
+  const auto reg = operand.to_register_ref();
+  if (!reg || reg->cls != RegClass::VGPR)
+    return {lo, lo};
+
+  const Operand hi_operand = packed_register_dword_offset(operand, 2);
+  return {lo, amdgpu::RegisterAccess(wf).read_lane64(hi_operand, lane)};
+}
+
+PkU32Pair read_pk_u32_pair(const Operand &operand, const amdgpu::Wavefront &wf, uint32_t lane) {
+  // GFX12+ single-SGPR-read operands read the first SGPR and replicate it.
+  // VGPRs and 64-bit special registers such as VCC and EXEC remain pairs.
+  const auto reg = operand.to_register_ref();
+  if (reg && reg->cls == RegClass::SGPR) {
+    const uint32_t value = amdgpu::RegisterAccess(wf).read_lane(operand, lane);
+    return {value, value};
+  }
+  const auto pair = amdgpu::RegisterAccess(wf).read_lane_pair32(operand, lane);
+  return {pair.lo, pair.hi};
+}
+
+void write_pk_u64_pair(const Operand &operand, amdgpu::Wavefront &wf, uint32_t lane,
+                       PkU64Pair value) {
+  const Operand hi_operand = packed_register_dword_offset(operand, 2);
+  amdgpu::RegisterAccess access(wf);
+  access.write_lane64(operand, lane, value.lo);
+  access.write_lane64(hi_operand, lane, value.hi);
+}
+
 uint16_t read_fma_mix_f16_bits(uint32_t raw, uint32_t src_selector, bool high_half) {
   switch (src_selector) {
   case OpSelSrc::OPR_SRC_FLOAT_HALF:
@@ -2945,6 +2991,33 @@ void VWmmaF3232x16x128F4Vop3p::execute_impl(amdgpu::Wavefront &wf) {
                         amdgpu::wmma_c_modifier(inst_.neg, inst_.neg_hi));
 }
 
+void VPkLshlAddU64Vop3p::execute_impl(amdgpu::Wavefront &wf) {
+  uint64_t exec = wf.exec();
+  std::array<PkU64Pair, 64> results{};
+  for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) {
+    if (!(exec & (1ULL << lane)))
+      continue;
+    const auto values = read_pk_u64_pair(src0, wf, lane);
+    const auto shifts = read_pk_u32_pair(src1, wf, lane);
+    const auto addends = read_pk_u64_pair(src2, wf, lane);
+    if (shifts.lo > 4u || shifts.hi > 4u) {
+      wf.report_instruction_execution_error(
+          amdgpu::InstructionExecutionError::UnsupportedOperandValue);
+      return;
+    }
+    const uint64_t result_lo =
+        amdgpu::lshl_masked(values.lo, static_cast<uint64_t>(shifts.lo)) + addends.lo;
+    const uint64_t result_hi =
+        amdgpu::lshl_masked(values.hi, static_cast<uint64_t>(shifts.hi)) + addends.hi;
+    results[lane] = {result_lo, result_hi};
+  }
+  for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) {
+    if (!(exec & (1ULL << lane)))
+      continue;
+    write_pk_u64_pair(vdst, wf, lane, results[lane]);
+  }
+}
+
 void VWmmaScaleF32Vop3px2::execute_impl(amdgpu::Wavefront &wf) {
   auto &cu = wf.cu();
   uint32_t vb = wf.vgpr_alloc().base;

@@ -60,6 +60,16 @@ constexpr uint32_t kPrivilegedStatusBit = 1u << 5;
 
 bool is_privileged(const Wavefront &wf) { return (wf.status_raw() & kPrivilegedStatusBit) != 0; }
 
+std::string_view instruction_execution_error_name(InstructionExecutionError error) {
+  switch (error) {
+  case InstructionExecutionError::None:
+    return "none";
+  case InstructionExecutionError::UnsupportedOperandValue:
+    return "unsupported operand value";
+  }
+  return "unknown instruction execution error";
+}
+
 uint32_t pack_barrier_state(uint32_t member_count, uint32_t signal_count,
                             uint32_t allocation_blocks = 0) {
   return 1u | ((member_count & 0x7fu) << 4) | ((signal_count & 0x7fu) << 16) |
@@ -884,6 +894,19 @@ void ComputeUnitCore::issue_instruction(Wavefront *active) {
 
   execute_instruction(inst, *active);
 
+  if (active->instruction_execution_failed()) {
+    const InstructionExecutionError error = active->instruction_execution_error();
+    const std::string failure = std::format("CU {}: wf{} could not execute {} at pc={:#x}: {}",
+                                            this->name(), active->wf_id(), inst->mnemonic(),
+                                            active->pc, instruction_execution_error_name(error));
+    util::Logger::warn(failure);
+    if (auto *sim_engine = this->engine())
+      sim_engine->request_exit(failure, /*code=*/1);
+    active->halt();
+    delete inst;
+    return;
+  }
+
   // A terminating instruction (s_endpgm with no pending waits) halts the wave
   // inside execute_instruction, which frees and resets its slot. Its registers,
   // pc, and allocations are now zeroed, so the after-execute hook, result logging,