From 14a3c4ddcc00a8c123661109a16ba3a0a227cc3e Mon Sep 17 00:00:00 2001 From: Jakub Kuderski Date: Sun, 1 Nov 2026 17:49:22 -0500 Subject: [PATCH] [rocjitsu] Compact generated operand storage Use one byte for literal widening and group generated operand fields by alignment across all ten AMDGPU architectures. On x86-64, shrink generated operands from 80 to 64 bytes on all ten targets. For CDNA5, a scalar add with carry drops from 536 to 472 bytes, a VOP3 move from 400 to 368 bytes, and VOPD from 992 to 864 bytes. Measure warm userspace CPU counters on a Threadripper PRO 9995WX with GCC 16 Release/O3 and x86-64-v4, using one simulation worker on one CPU: - Eleven distinct ML inputs, two paired sweeps: cycles -0.67%, L2 data-request misses -3.09%, instructions -0.06%. - Seven small kernels on CDNA3/4/5, three paired sweeps: cycles -0.21%. By target: CDNA3 -0.11%, CDNA4 +0.65%, CDNA5 -1.16%. Cycles vary by target despite the same operand size reduction: CDNA5 ML cycles decrease 1.59%, while CDNA4 increases 0.47%. These measurements do not isolate the cause of that difference. Use equal-cell geometric means. Measurements are unreserved, with CPU affinity, a 2.5 GHz frequency policy, ASLR disabled, wait checking enabled and an exclusive smoke flock. --- .../rocjitsu/lib/python/amdisa/codegen/_generator.py | 7 ++++--- .../rocjitsu/isa/arch/amdgpu/generated/cdna1/operand.h | 7 ++++--- .../rocjitsu/isa/arch/amdgpu/generated/cdna2/operand.h | 7 ++++--- .../rocjitsu/isa/arch/amdgpu/generated/cdna3/operand.h | 7 ++++--- .../rocjitsu/isa/arch/amdgpu/generated/cdna4/operand.h | 7 ++++--- .../rocjitsu/isa/arch/amdgpu/generated/cdna5/operand.h | 7 ++++--- .../rocjitsu/isa/arch/amdgpu/generated/rdna1/operand.h | 7 ++++--- .../rocjitsu/isa/arch/amdgpu/generated/rdna2/operand.h | 7 ++++--- .../rocjitsu/isa/arch/amdgpu/generated/rdna3/operand.h | 7 ++++--- .../isa/arch/amdgpu/generated/rdna3_5/operand.h | 7 ++++--- .../rocjitsu/isa/arch/amdgpu/generated/rdna4/operand.h | 7 ++++--- emulation/rocjitsu/tests/decode_smoke_test.cpp | 10 +++++++--- 12 files changed, 51 insertions(+), 36 deletions(-) diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py index 867192956ecc..41f28ee63c31 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py @@ -16746,7 +16746,7 @@ def gen_operand(self) -> None: cgen.Line( f'{operand_base_decl}' 'public:\n' - ' enum class Literal32Widening { ZeroExtend, SignExtend, Replicate32, F64HighBits };\n' + ' enum class Literal32Widening : uint8_t { ZeroExtend, SignExtend, Replicate32, F64HighBits };\n' ' static constexpr bool kStaticRegisterAccess = true;\n' f'{operand_ctor_decl}' ' Operand(int size_bits, OperandType opr_type, int encoding_value,\n' @@ -16761,11 +16761,12 @@ def gen_operand(self) -> None: f'{execution_backend_public_decl}' f'{execution_decls}' ' uint64_t widened_literal32_value() const;\n' + ' // Pack literal state with the byte-sized widening enum to keep operands at 64 bytes.\n' + ' uint64_t literal64_value_ = 0;\n' + ' std::optional literal32_widening_;\n' ' uint16_t literal16_display_value_ = 0;\n' ' bool has_literal16_display_ = false;\n' - ' uint64_t literal64_value_ = 0;\n' ' bool has_literal64_ = false;\n' - ' std::optional literal32_widening_;\n' f'{packed_16bit_field}' '};' ) diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/operand.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/operand.h index 60105ed12d5e..f6309e98718b 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/operand.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna1/operand.h @@ -18,7 +18,7 @@ namespace cdna1 { class Operand final : public IsaOperand { public: - enum class Literal32Widening { ZeroExtend, SignExtend, Replicate32, F64HighBits }; + enum class Literal32Widening : uint8_t { ZeroExtend, SignExtend, Replicate32, F64HighBits }; static constexpr bool kStaticRegisterAccess = true; Operand(int size_bits, OperandType opr_type, int encoding_value); Operand(int size_bits, OperandType opr_type, int encoding_value, uint16_t literal16_display_value, @@ -130,11 +130,12 @@ class Operand final : public IsaOperand { void simd_notify_write_mut_exec(amdgpu::Wavefront &, uint64_t, uint8_t) const; void simd_notify_write64_mut_exec(amdgpu::Wavefront &, uint64_t, uint8_t) const; uint64_t widened_literal32_value() const; + // Pack literal state with the byte-sized widening enum to keep operands at 64 bytes. + uint64_t literal64_value_ = 0; + std::optional literal32_widening_; uint16_t literal16_display_value_ = 0; bool has_literal16_display_ = false; - uint64_t literal64_value_ = 0; bool has_literal64_ = false; - std::optional literal32_widening_; }; } // namespace cdna1 diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/operand.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/operand.h index 372411a22b11..7a8076553ffd 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/operand.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna2/operand.h @@ -18,7 +18,7 @@ namespace cdna2 { class Operand final : public IsaOperand { public: - enum class Literal32Widening { ZeroExtend, SignExtend, Replicate32, F64HighBits }; + enum class Literal32Widening : uint8_t { ZeroExtend, SignExtend, Replicate32, F64HighBits }; static constexpr bool kStaticRegisterAccess = true; Operand(int size_bits, OperandType opr_type, int encoding_value); Operand(int size_bits, OperandType opr_type, int encoding_value, uint16_t literal16_display_value, @@ -130,11 +130,12 @@ class Operand final : public IsaOperand { void simd_notify_write_mut_exec(amdgpu::Wavefront &, uint64_t, uint8_t) const; void simd_notify_write64_mut_exec(amdgpu::Wavefront &, uint64_t, uint8_t) const; uint64_t widened_literal32_value() const; + // Pack literal state with the byte-sized widening enum to keep operands at 64 bytes. + uint64_t literal64_value_ = 0; + std::optional literal32_widening_; uint16_t literal16_display_value_ = 0; bool has_literal16_display_ = false; - uint64_t literal64_value_ = 0; bool has_literal64_ = false; - std::optional literal32_widening_; }; } // namespace cdna2 diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/operand.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/operand.h index 8fb769d11b06..55e48275a941 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/operand.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna3/operand.h @@ -18,7 +18,7 @@ namespace cdna3 { class Operand final : public IsaOperand { public: - enum class Literal32Widening { ZeroExtend, SignExtend, Replicate32, F64HighBits }; + enum class Literal32Widening : uint8_t { ZeroExtend, SignExtend, Replicate32, F64HighBits }; static constexpr bool kStaticRegisterAccess = true; Operand(int size_bits, OperandType opr_type, int encoding_value); Operand(int size_bits, OperandType opr_type, int encoding_value, uint16_t literal16_display_value, @@ -130,11 +130,12 @@ class Operand final : public IsaOperand { void simd_notify_write_mut_exec(amdgpu::Wavefront &, uint64_t, uint8_t) const; void simd_notify_write64_mut_exec(amdgpu::Wavefront &, uint64_t, uint8_t) const; uint64_t widened_literal32_value() const; + // Pack literal state with the byte-sized widening enum to keep operands at 64 bytes. + uint64_t literal64_value_ = 0; + std::optional literal32_widening_; uint16_t literal16_display_value_ = 0; bool has_literal16_display_ = false; - uint64_t literal64_value_ = 0; bool has_literal64_ = false; - std::optional literal32_widening_; }; } // namespace cdna3 diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/operand.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/operand.h index 018b4ad936c9..99711d6f90e4 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/operand.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna4/operand.h @@ -18,7 +18,7 @@ namespace cdna4 { class Operand final : public IsaOperand { public: - enum class Literal32Widening { ZeroExtend, SignExtend, Replicate32, F64HighBits }; + enum class Literal32Widening : uint8_t { ZeroExtend, SignExtend, Replicate32, F64HighBits }; static constexpr bool kStaticRegisterAccess = true; Operand(int size_bits, OperandType opr_type, int encoding_value); Operand(int size_bits, OperandType opr_type, int encoding_value, uint16_t literal16_display_value, @@ -130,11 +130,12 @@ class Operand final : public IsaOperand { void simd_notify_write_mut_exec(amdgpu::Wavefront &, uint64_t, uint8_t) const; void simd_notify_write64_mut_exec(amdgpu::Wavefront &, uint64_t, uint8_t) const; uint64_t widened_literal32_value() const; + // Pack literal state with the byte-sized widening enum to keep operands at 64 bytes. + uint64_t literal64_value_ = 0; + std::optional literal32_widening_; uint16_t literal16_display_value_ = 0; bool has_literal16_display_ = false; - uint64_t literal64_value_ = 0; bool has_literal64_ = false; - std::optional literal32_widening_; }; } // namespace cdna4 diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/operand.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/operand.h index 2969807980dd..60b8104a17dc 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/operand.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/cdna5/operand.h @@ -18,7 +18,7 @@ namespace cdna5 { class Operand final : public IsaOperand { public: - enum class Literal32Widening { ZeroExtend, SignExtend, Replicate32, F64HighBits }; + enum class Literal32Widening : uint8_t { ZeroExtend, SignExtend, Replicate32, F64HighBits }; static constexpr bool kStaticRegisterAccess = true; Operand(int size_bits, OperandType opr_type, int encoding_value, bool packed_16bit_source = false, bool packed_16bit_dst = false); @@ -133,11 +133,12 @@ class Operand final : public IsaOperand { void simd_notify_write_mut_exec(amdgpu::Wavefront &, uint64_t, uint8_t) const; void simd_notify_write64_mut_exec(amdgpu::Wavefront &, uint64_t, uint8_t) const; uint64_t widened_literal32_value() const; + // Pack literal state with the byte-sized widening enum to keep operands at 64 bytes. + uint64_t literal64_value_ = 0; + std::optional literal32_widening_; uint16_t literal16_display_value_ = 0; bool has_literal16_display_ = false; - uint64_t literal64_value_ = 0; bool has_literal64_ = false; - std::optional literal32_widening_; bool packed_16bit_source_ = false; bool packed_16bit_dst_ = false; std::optional resolved_vgpr_offset_exec(const amdgpu::Wavefront &wf) const; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/operand.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/operand.h index a2aa5695e6d7..5ea1cf470a9c 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/operand.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna1/operand.h @@ -18,7 +18,7 @@ namespace rdna1 { class Operand final : public IsaOperand { public: - enum class Literal32Widening { ZeroExtend, SignExtend, Replicate32, F64HighBits }; + enum class Literal32Widening : uint8_t { ZeroExtend, SignExtend, Replicate32, F64HighBits }; static constexpr bool kStaticRegisterAccess = true; Operand(int size_bits, OperandType opr_type, int encoding_value); Operand(int size_bits, OperandType opr_type, int encoding_value, uint16_t literal16_display_value, @@ -130,11 +130,12 @@ class Operand final : public IsaOperand { void simd_notify_write_mut_exec(amdgpu::Wavefront &, uint64_t, uint8_t) const; void simd_notify_write64_mut_exec(amdgpu::Wavefront &, uint64_t, uint8_t) const; uint64_t widened_literal32_value() const; + // Pack literal state with the byte-sized widening enum to keep operands at 64 bytes. + uint64_t literal64_value_ = 0; + std::optional literal32_widening_; uint16_t literal16_display_value_ = 0; bool has_literal16_display_ = false; - uint64_t literal64_value_ = 0; bool has_literal64_ = false; - std::optional literal32_widening_; }; } // namespace rdna1 diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/operand.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/operand.h index a7d9b6b25dae..0dd6655d41ae 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/operand.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna2/operand.h @@ -18,7 +18,7 @@ namespace rdna2 { class Operand final : public IsaOperand { public: - enum class Literal32Widening { ZeroExtend, SignExtend, Replicate32, F64HighBits }; + enum class Literal32Widening : uint8_t { ZeroExtend, SignExtend, Replicate32, F64HighBits }; static constexpr bool kStaticRegisterAccess = true; Operand(int size_bits, OperandType opr_type, int encoding_value); Operand(int size_bits, OperandType opr_type, int encoding_value, uint16_t literal16_display_value, @@ -130,11 +130,12 @@ class Operand final : public IsaOperand { void simd_notify_write_mut_exec(amdgpu::Wavefront &, uint64_t, uint8_t) const; void simd_notify_write64_mut_exec(amdgpu::Wavefront &, uint64_t, uint8_t) const; uint64_t widened_literal32_value() const; + // Pack literal state with the byte-sized widening enum to keep operands at 64 bytes. + uint64_t literal64_value_ = 0; + std::optional literal32_widening_; uint16_t literal16_display_value_ = 0; bool has_literal16_display_ = false; - uint64_t literal64_value_ = 0; bool has_literal64_ = false; - std::optional literal32_widening_; }; } // namespace rdna2 diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/operand.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/operand.h index ff99a8a4ceb7..8facdf3b3693 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/operand.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3/operand.h @@ -18,7 +18,7 @@ namespace rdna3 { class Operand final : public IsaOperand { public: - enum class Literal32Widening { ZeroExtend, SignExtend, Replicate32, F64HighBits }; + enum class Literal32Widening : uint8_t { ZeroExtend, SignExtend, Replicate32, F64HighBits }; static constexpr bool kStaticRegisterAccess = true; Operand(int size_bits, OperandType opr_type, int encoding_value, bool packed_16bit_source = false, bool packed_16bit_dst = false); @@ -133,11 +133,12 @@ class Operand final : public IsaOperand { void simd_notify_write_mut_exec(amdgpu::Wavefront &, uint64_t, uint8_t) const; void simd_notify_write64_mut_exec(amdgpu::Wavefront &, uint64_t, uint8_t) const; uint64_t widened_literal32_value() const; + // Pack literal state with the byte-sized widening enum to keep operands at 64 bytes. + uint64_t literal64_value_ = 0; + std::optional literal32_widening_; uint16_t literal16_display_value_ = 0; bool has_literal16_display_ = false; - uint64_t literal64_value_ = 0; bool has_literal64_ = false; - std::optional literal32_widening_; bool packed_16bit_source_ = false; bool packed_16bit_dst_ = false; std::optional resolved_vgpr_offset_exec(const amdgpu::Wavefront &wf) const; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/operand.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/operand.h index 0d513689b081..9f958fb56735 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/operand.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna3_5/operand.h @@ -18,7 +18,7 @@ namespace rdna3_5 { class Operand final : public IsaOperand { public: - enum class Literal32Widening { ZeroExtend, SignExtend, Replicate32, F64HighBits }; + enum class Literal32Widening : uint8_t { ZeroExtend, SignExtend, Replicate32, F64HighBits }; static constexpr bool kStaticRegisterAccess = true; Operand(int size_bits, OperandType opr_type, int encoding_value, bool packed_16bit_source = false, bool packed_16bit_dst = false); @@ -133,11 +133,12 @@ class Operand final : public IsaOperand { void simd_notify_write_mut_exec(amdgpu::Wavefront &, uint64_t, uint8_t) const; void simd_notify_write64_mut_exec(amdgpu::Wavefront &, uint64_t, uint8_t) const; uint64_t widened_literal32_value() const; + // Pack literal state with the byte-sized widening enum to keep operands at 64 bytes. + uint64_t literal64_value_ = 0; + std::optional literal32_widening_; uint16_t literal16_display_value_ = 0; bool has_literal16_display_ = false; - uint64_t literal64_value_ = 0; bool has_literal64_ = false; - std::optional literal32_widening_; bool packed_16bit_source_ = false; bool packed_16bit_dst_ = false; std::optional resolved_vgpr_offset_exec(const amdgpu::Wavefront &wf) const; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/operand.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/operand.h index 8309abdf943c..3b4a687fe013 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/operand.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/generated/rdna4/operand.h @@ -18,7 +18,7 @@ namespace rdna4 { class Operand final : public IsaOperand { public: - enum class Literal32Widening { ZeroExtend, SignExtend, Replicate32, F64HighBits }; + enum class Literal32Widening : uint8_t { ZeroExtend, SignExtend, Replicate32, F64HighBits }; static constexpr bool kStaticRegisterAccess = true; Operand(int size_bits, OperandType opr_type, int encoding_value, bool packed_16bit_source = false, bool packed_16bit_dst = false); @@ -133,11 +133,12 @@ class Operand final : public IsaOperand { void simd_notify_write_mut_exec(amdgpu::Wavefront &, uint64_t, uint8_t) const; void simd_notify_write64_mut_exec(amdgpu::Wavefront &, uint64_t, uint8_t) const; uint64_t widened_literal32_value() const; + // Pack literal state with the byte-sized widening enum to keep operands at 64 bytes. + uint64_t literal64_value_ = 0; + std::optional literal32_widening_; uint16_t literal16_display_value_ = 0; bool has_literal16_display_ = false; - uint64_t literal64_value_ = 0; bool has_literal64_ = false; - std::optional literal32_widening_; bool packed_16bit_source_ = false; bool packed_16bit_dst_ = false; std::optional resolved_vgpr_offset_exec(const amdgpu::Wavefront &wf) const; diff --git a/emulation/rocjitsu/tests/decode_smoke_test.cpp b/emulation/rocjitsu/tests/decode_smoke_test.cpp index 9e9c62ddeb87..327935dd1e94 100644 --- a/emulation/rocjitsu/tests/decode_smoke_test.cpp +++ b/emulation/rocjitsu/tests/decode_smoke_test.cpp @@ -109,10 +109,14 @@ constexpr uint32_t make_cdna1_sop1(uint32_t sdst, uint32_t ssrc0) { return 0xBE800000u | ((sdst & 0x7Fu) << 16) | (ssrc0 & 0xFFu); } -TEST(OperandLayoutTest, DeferredSelectorStateFitsExistingPadding) { +TEST(OperandLayoutTest, GeneratedOperandsUse64Bytes) { + EXPECT_EQ(sizeof(Instruction), 192u); EXPECT_EQ(sizeof(Operand), 32u); - EXPECT_EQ(sizeof(cdna5::Operand), 80u); - EXPECT_EQ(sizeof(cdna5::VAddF32Vop3), 480u); + // Cover generator profiles with and without packed 16-bit state. + EXPECT_EQ(sizeof(cdna4::Operand), 64u); + EXPECT_EQ(sizeof(cdna5::Operand), 64u); + EXPECT_EQ(sizeof(cdna5::VAddF32Vop3), 432u); + EXPECT_EQ(sizeof(cdna5::Vopd), 864u); } TEST(OperandLayoutTest, DecodedVectorRegistersExcludeScalarAndLiteralSelectors) {