diff --git a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py index 1eb39249feeb..1b0d00953e62 100644 --- a/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py +++ b/emulation/rocjitsu/lib/python/amdisa/codegen/_generator.py @@ -1924,6 +1924,12 @@ def gen_execution_backend(self) -> None: f' &execute_with_backend<{class_name}>,\n' for class_name in self._split_execution_classes ) + # Generated executors follow Instruction's decoded-reuse contract: + # non-memory instructions without DynamicInstState can execute again + # on another wave. Read values and EXEC from the current context, + # restore temporary operand delegates, and assign per-execution flags + # on every call. Persistent per-issue state belongs in DynamicInstState, + # not decoded encoding or operand members. source = textwrap.dedent(f'''\ {CppFile._prologue_comment()} #include "{generated_arch}/execution_backend.h" diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/decoder.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/decoder.h index b9ddc6dabcc7..1ae7121401fe 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/decoder.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/decoder.h @@ -30,8 +30,9 @@ struct IsaExecutionBackend; /// By default, decoded instructions are heap-allocated. Call /// ``enable_pool()`` to route Instruction::operator new/delete through /// the decoder's O(1) free-list pool. Only enable the pool when all -/// decoded instructions will be deleted before the decoder is destroyed -/// (e.g., the ComputeUnit simulation loop). +/// decoded instructions will be deleted on the bound thread before the decoder +/// is destroyed. CU execution uses heap storage because decoded instructions +/// can survive issue quanta and move between workers. class Decoder { public: using Pool = util::ArenaAlloc<512, 128>; @@ -97,8 +98,8 @@ class Decoder { /// /// When active, Instruction::operator new/delete route through the /// decoder's pool for O(1) alloc/free. Only enable when the caller - /// guarantees all instructions will be deleted before the decoder - /// is destroyed (e.g., the ComputeUnit hot path). + /// guarantees all instructions will be deleted on the bound thread + /// before the decoder is destroyed. CU execution uses heap storage. void enable_pool() { activate_pool([](void *p, size_t s) -> void * { return static_cast(p)->allocate(s); }, [](void *p, void *ptr) { static_cast(p)->deallocate(ptr); }, &pool_); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/instruction.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/instruction.h index 28b09770026c..65eaf6b9576c 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/instruction.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/instruction.h @@ -112,9 +112,9 @@ class Instruction : public util::IListNode d) { data_ = std::move(d); } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/async_scoreboard.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/async_scoreboard.h index 32367b4d2e54..dc3df00e3525 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/async_scoreboard.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/async_scoreboard.h @@ -119,8 +119,8 @@ class AsyncInstructionQueue { }; /// @brief A bounded same-wave execution window. -/// @details Draining before CU rescheduling keeps decoder pools, wave storage -/// and data caches owned by the issuing thread. +/// @details Draining before CU rescheduling keeps wave storage and data caches +/// stable until asynchronous work finishes. class AsyncInstructionWindow { using Access = async_execution::Access; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_unit.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_unit.cpp index 62a63f41c03c..08bbe0b3dd90 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_unit.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_unit.cpp @@ -165,10 +165,9 @@ ComputeUnitCore::ComputeUnitCore(std::string name, const Config &config, GpuMemo inst_cache_.set_l2(l2_); - // Enable pool allocation for the hot decode-execute path. - // Instructions decoded during step() are always deleted before the CU - // (and its decoder) are destroyed, so pool allocation is safe here. - decoder_->enable_pool(); + // Decoded instructions use heap storage: cache entries and asynchronous + // work can outlive an issue quantum and move between execution workers. + // Do not install this decoder's pool on the constructing thread. wfs_.resize(config.num_wf_slots); sgpr_file_.init(config.num_wf_slots * config.sgprs_per_wf, config.sgprs_per_wf); @@ -1510,7 +1509,9 @@ template active->trace_inst_count_++; util::StringDiagnostic decode_error; - DecodeResult decoded = decoder_->decode(words, decode_error.emitter()); + const bool reuse_decoded = !debug_active(); + DecodeResult decoded = decoded_inst_cache_.decode(*decoder_, active->pc, vmid, words, + decode_error.emitter(), reuse_decoded); if (decoded.failed()) { drain_async_window(); util::Logger::vm("CU ", this->name(), ": wf", active->wf_id(), " HALT(decode rejection) pc=0x", @@ -1526,6 +1527,8 @@ template return; } Instruction *inst = decoded.value().get(); + DecodedInstructionCache::ScopedReturn return_decoded(decoded_inst_cache_, decoded.value(), + active->pc, vmid, words, reuse_decoded); int inst_size_signed = inst->size(); assert(inst_size_signed > 0 && "instruction size must be positive"); @@ -1726,6 +1729,7 @@ template }(); if (execution_result.failed()) [[unlikely]] { + return_decoded.discard(); if constexpr (EnableAsync) { if (window) window->drain(); @@ -1748,7 +1752,7 @@ template // pc, and allocations are now zeroed, so the after-execute hook, result logging, // and pc-advance below must not run on the dead slot. The dedicated // onAmdgpuWavefrontHalted hook already fired (with live state) from halt(). - // s_endpgm is never a memory op, so just reclaim the decoded instruction. + // The completed s_endpgm can return to the decoded cache on scope exit. // // Note the intentional asymmetry: an s_endpgm that defers to ENDING (pending // memory waits) is NOT halted here, so it DOES fire onAmdgpuAfterExecuteInstruction @@ -1930,8 +1934,6 @@ template return; } } - } else { - decoded.value().reset(); } active->pc += inst_size; diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_unit.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_unit.h index 1fbc6789cae8..ddfd7a6d1bb7 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_unit.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/compute_unit.h @@ -11,6 +11,7 @@ #include "rocjitsu/isa/decoder.h" #include "rocjitsu/isa/instruction.h" #include "rocjitsu/vm/amdgpu/cluster_lds_multicast.h" +#include "rocjitsu/vm/amdgpu/decoded_instruction_cache.h" #include "rocjitsu/vm/amdgpu/gpu_memory.h" #include "rocjitsu/vm/amdgpu/gpu_vm.h" #include "rocjitsu/vm/amdgpu/instruction_cache.h" @@ -1056,7 +1057,7 @@ class RJ_API_TYPE_EXPORT ComputeUnitCore : public simdojo::CompositeComponent { void replace_decoder_for_test(std::unique_ptr decoder) { assert(decoder != nullptr); assert(!has_active_wfs()); - decoder->enable_pool(); + decoded_inst_cache_.clear(); decoder_ = std::move(decoder); } @@ -1291,6 +1292,7 @@ class RJ_API_TYPE_EXPORT ComputeUnitCore : public simdojo::CompositeComponent { L1ScalarCache l1_scalar_; L1VectorCache l1_vector_; InstructionCache inst_cache_; + DecodedInstructionCache decoded_inst_cache_; /// @brief Debug attach/detach transitions seen by set_debug_active(). std::atomic inst_cache_debug_epoch_{0}; /// @brief The epoch this CU's thread has already invalidated the I$ for. diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/decoded_instruction_cache.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/decoded_instruction_cache.h new file mode 100644 index 000000000000..7b0eade759bd --- /dev/null +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/decoded_instruction_cache.h @@ -0,0 +1,133 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#pragma once + +#include "rocjitsu/isa/decoder.h" +#include "rocjitsu/isa/instruction.h" + +#include +#include +#include +#include +#include +#include + +namespace rocjitsu::amdgpu { + +/// @brief Reuse completed instructions on one CU's issue path. +/// +/// @details Fetchability and the byte I$ remain authoritative: lookups compare +/// all fetched words, including literals, so code maintenance and debugger +/// writes cannot reuse stale decodes. +/// Instructions that borrow the fetch buffer cannot be retained. Entries use +/// heap storage because a CU can run on different workers. Re-execution follows +/// the stateless Instruction::execute contract. +class DecodedInstructionCache { +public: + // Bounded host-speed choice; direct indexing requires a power of two. + static constexpr size_t kNumEntries = 256; + static_assert(kNumEntries != 0 && (kNumEntries & (kNumEntries - 1)) == 0); + // Match the four-word CU instruction-fetch window, including literals. + static constexpr size_t kNumWords = 4; + using Words = std::span; + + ~DecodedInstructionCache() { clear(); } + + /// @brief Decode from freshly fetched words, or take the matching cached object. + /// + /// @details The CU uses heap storage for every miss, including uncached + /// memory operations and debugger execution. The allocation guard also + /// bypasses an unrelated decoder pool installed by the caller. + /// @param enabled False bypasses reuse and table allocation while debugging. + DecodeResult decode(Decoder &decoder, uint64_t pc, uint32_t vmid, Words words, + const DecodeErrorEmitter &emit_error, bool enabled = true) { + if (enabled && !entries_) + entries_ = std::make_unique>(); + if (enabled && entries_) { + auto &entry = (*entries_)[index(pc)]; + if (entry.instruction && entry.pc == pc && entry.vmid == vmid && + std::equal(words.begin(), words.end(), entry.words.begin())) + return std::move(entry.instruction); + } + Instruction::ScopedHeapAllocation heap_allocation; + return decoder.decode(words.data(), emit_error); + } + + /// @brief Return only completed, stateless instructions. + /// + /// @details Construct immediately after decode() with the same key and policy. + /// Memory pipelines and async execution take ownership by releasing/moving + /// the caller's unique_ptr, so their in-flight instructions cannot become + /// cache entries. + class ScopedReturn { + public: + ScopedReturn(DecodedInstructionCache &cache, std::unique_ptr &instruction, + uint64_t pc, uint32_t vmid, Words words, bool enabled = true) + : cache_(cache), instruction_(instruction), pc_(pc), vmid_(vmid), words_(words), + enabled_(enabled), exceptions_(std::uncaught_exceptions()) {} + + ~ScopedReturn() { + if (enabled_ && instruction_ && !instruction_->is_memory_op() && !instruction_->data() && + !borrows_fetch_buffer(*instruction_, words_) && std::uncaught_exceptions() == exceptions_) + cache_.put(pc_, vmid_, words_, instruction_); + } + + ScopedReturn(const ScopedReturn &) = delete; + ScopedReturn &operator=(const ScopedReturn &) = delete; + + /// @brief Exclude a failed execution from reuse. + void discard() { enabled_ = false; } + + private: + DecodedInstructionCache &cache_; + std::unique_ptr &instruction_; + uint64_t pc_; + uint32_t vmid_; + Words words_; + bool enabled_; + int exceptions_; + }; + + /// @brief Drop entries before replacing the decoder/target, or destroying this CU. + void clear() { + Instruction::ScopedHeapAllocation heap_allocation; + entries_.reset(); + } + +private: + struct Entry { + uint64_t pc = 0; + uint32_t vmid = 0; + std::array words{}; + std::unique_ptr instruction; + }; + + static size_t index(uint64_t pc) { return (pc / sizeof(uint32_t)) & (kNumEntries - 1); } + + static bool borrows_fetch_buffer(const Instruction &instruction, Words words) { + const auto raw = reinterpret_cast(instruction.raw_encoding()); + const auto begin = reinterpret_cast(words.data()); + return raw >= begin && raw - begin < words.size_bytes(); + } + + void put(uint64_t pc, uint32_t vmid, Words words, std::unique_ptr &instruction) { + // A decode miss allocates the table before execution; returning an + // instruction during stack unwinding must not allocate. + if (!entries_) + return; + auto &entry = (*entries_)[index(pc)]; + if (entry.instruction) { + Instruction::ScopedHeapAllocation heap_allocation; + entry.instruction.reset(); + } + entry.pc = pc; + entry.vmid = vmid; + std::copy(words.begin(), words.end(), entry.words.begin()); + entry.instruction = std::move(instruction); + } + + std::unique_ptr> entries_; +}; + +} // namespace rocjitsu::amdgpu diff --git a/emulation/rocjitsu/tests/instruction_cache_test.cpp b/emulation/rocjitsu/tests/instruction_cache_test.cpp index 45fa8e065d05..74055a00872a 100644 --- a/emulation/rocjitsu/tests/instruction_cache_test.cpp +++ b/emulation/rocjitsu/tests/instruction_cache_test.cpp @@ -2,8 +2,13 @@ // SPDX-License-Identifier: MIT #include "rocjitsu/code/rj_code.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna4/encodings.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna4/machine_insts.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna4/opcodes.h" +#include "rocjitsu/isa/arch/amdgpu/shared/instruction_encoding.h" #include "rocjitsu/kmd/linux/kfd_process.h" #include "rocjitsu/vm/amdgpu/compute_unit.h" +#include "rocjitsu/vm/amdgpu/decoded_instruction_cache.h" #include "rocjitsu/vm/amdgpu/gpu_memory.h" #include "rocjitsu/vm/amdgpu/instruction_cache.h" #include "rocjitsu/vm/amdgpu/l2_cache.h" @@ -20,6 +25,7 @@ #include #include #include +#include #include namespace { @@ -360,6 +366,360 @@ class CuFixture { std::unique_ptr cu_; }; +class CountingDecoder : public rocjitsu::Decoder { +public: + explicit CountingDecoder(rj_code_arch_t arch = ROCJITSU_CODE_ARCH_CDNA4) + : decoder_(rocjitsu::Decoder::create(arch)) {} + + rocjitsu::DecodeResult decode(const rj_code_binary_inst_t *words, + const rocjitsu::DecodeErrorEmitter &emit_error) override { + ++calls; + return decoder_->decode(words, emit_error); + } + size_t max_instruction_words() const override { return decoder_->max_instruction_words(); } + + unsigned calls = 0; + +private: + std::unique_ptr decoder_; +}; + +TEST(DecodedInstructionCacheCuTest, ConstructionAndDecoderReplacementPreserveForeignPool) { + rocjitsu::Instruction::ScopedHeapAllocation restore_allocator; + CountingDecoder foreign_decoder; + foreign_decoder.enable_pool(); + const auto alloc_fn = rocjitsu::Instruction::alloc_fn_; + const auto dealloc_fn = rocjitsu::Instruction::dealloc_fn_; + const auto pool = rocjitsu::Instruction::alloc_pool_; + ASSERT_NE(pool, nullptr); + const auto expect_foreign_pool = [&] { + EXPECT_EQ(rocjitsu::Instruction::alloc_fn_, alloc_fn); + EXPECT_EQ(rocjitsu::Instruction::dealloc_fn_, dealloc_fn); + EXPECT_EQ(rocjitsu::Instruction::alloc_pool_, pool); + }; + { + CuFixture fixture("decoded_foreign_pool"); + expect_foreign_pool(); + fixture.cu()->replace_decoder_for_test(std::make_unique()); + expect_foreign_pool(); + fixture.write_program(std::array{s_mov_b32_s0_imm(17), kSEndpgm}); + auto *wf = fixture.launch(1, 0); + ASSERT_NE(wf, nullptr); + fixture.cu()->step(); + EXPECT_EQ(fixture.read_s0(*wf), 17u); + expect_foreign_pool(); + fixture.cu()->step(); + expect_foreign_pool(); + } + expect_foreign_pool(); +} + +TEST(DecodedInstructionCacheCuTest, LoopDecodesOnceAndReadsCurrentRegisters) { + CuFixture fixture("decoded_loop"); + auto decoder = std::make_unique(); + auto *counter = decoder.get(); + fixture.cu()->replace_decoder_for_test(std::move(decoder)); + // s_mov_b32 s0, s1; s_branch -2. The branch revisits both decoded objects. + fixture.write_program(std::array{0xBE800001u, 0xBF82FFFEu}); + auto *wf = fixture.launch(1, 0); + ASSERT_NE(wf, nullptr); + for (uint32_t value = 1; value <= 10; ++value) { + fixture.cu()->write_sgpr(wf->sgpr_alloc().base + 1, value * 19); + fixture.cu()->step(); + EXPECT_EQ(fixture.read_s0(*wf), value * 19); + fixture.cu()->step(); + ASSERT_EQ(wf->pc, kCodeBase); + } + EXPECT_EQ(counter->calls, 2u); +} + +TEST(DecodedInstructionCacheCuTest, RewrittenLiteralAndDebuggerBypassUseFreshDecodes) { + CuFixture fixture("decoded_literal"); + auto decoder = std::make_unique(); + auto *counter = decoder.get(); + fixture.cu()->replace_decoder_for_test(std::move(decoder)); + fixture.write_program(std::array{0xBE8000FFu, 17u, kSEndpgm}); + auto *wf = fixture.launch(1, 0); + ASSERT_NE(wf, nullptr); + fixture.cu()->step(); + EXPECT_EQ(fixture.read_s0(*wf), 17u); + + fixture.memory().write32(kCodeBase + 4, 29u); + wf->pc = kCodeBase; + fixture.cu()->step(); + EXPECT_EQ(fixture.read_s0(*wf), 17u); + EXPECT_EQ(counter->calls, 1u); + + fixture.cu()->instruction_cache().invalidate_all(); + wf->pc = kCodeBase; + fixture.cu()->step(); + EXPECT_EQ(fixture.read_s0(*wf), 29u); + EXPECT_EQ(counter->calls, 2u); + + fixture.cu()->set_debug_active(true); + fixture.memory().write32(kCodeBase + 4, 43u); + wf->pc = kCodeBase; + fixture.cu()->step(); + EXPECT_EQ(fixture.read_s0(*wf), 43u); + fixture.cu()->set_debug_active(false); + wf->pc = kCodeBase; + fixture.cu()->step(); + EXPECT_EQ(fixture.read_s0(*wf), 43u); + EXPECT_EQ(counter->calls, 4u); +} + +TEST(DecodedInstructionCacheCuTest, VectorReadsCurrentExecAcrossDispatches) { + CuFixture fixture("decoded_vector", 2); + auto decoder = std::make_unique(); + auto *counter = decoder.get(); + fixture.cu()->replace_decoder_for_test(std::move(decoder)); + // v_mov_b32_e32 v0, v1; s_branch -2. + fixture.write_program(std::array{0x7E000301u, 0xBF82FFFEu}); + for (unsigned dispatch = 1; dispatch <= 3; ++dispatch) { + auto *wf = fixture.launch(dispatch, dispatch); + ASSERT_NE(wf, nullptr); + const auto base = wf->vgpr_alloc().base; + for (uint64_t mask : {~uint64_t{0}, uint64_t{0}, uint64_t{0x3333333333333333}}) { + wf->set_exec(mask); + for (unsigned lane = 0; lane < wf->wf_size(); ++lane) { + fixture.cu()->write_vgpr(base, lane, 0xDEAD0000u + lane); + fixture.cu()->write_vgpr(base + 1, lane, dispatch * 1000 + lane); + } + fixture.cu()->step(); + ASSERT_EQ(wf->pc, kCodeBase + 4); + for (unsigned lane = 0; lane < wf->wf_size(); ++lane) + EXPECT_EQ(fixture.cu()->read_vgpr(base, lane), + ((mask >> lane) & 1) ? dispatch * 1000 + lane : 0xDEAD0000u + lane); + fixture.cu()->step(); + ASSERT_EQ(wf->pc, kCodeBase); + } + wf->halt(); + } + EXPECT_EQ(counter->calls, 2u); +} + +TEST(DecodedInstructionCacheCuTest, DppDelegatesReadCurrentRegistersAndRestoreAcrossWaves) { + CuFixture fixture("decoded_dpp", 2); + auto *wf0 = fixture.launch(1, 0); + auto *wf1 = fixture.launch(1, 1); + ASSERT_NE(wf0, nullptr); + ASSERT_NE(wf1, nullptr); + ASSERT_NE(wf0->vgpr_alloc().base, wf1->vgpr_alloc().base); + CountingDecoder decoder; + amdgpu::DecodedInstructionCache cache; + // v_mov_b32 v0, v1 with DPP quad permutation [1, 0, 3, 2]. + rocjitsu::cdna4::Vop1VopDppMachineInst raw{}; + raw.src0 = amdgpu::SRC_DPP; + raw.op = rocjitsu::cdna4::kVMovB32Vop1; + raw.encoding = rocjitsu::cdna4::encoding::kVop1 >> 2; + raw.vdst = 0; + raw.vsrc0 = 1; + raw.dpp_ctrl = 0xB1; + raw.bound_ctrl = 1; + raw.bank_mask = 0xF; + raw.row_mask = 0xF; + std::array words{}; + std::memcpy(words.data(), &raw, sizeof(raw)); + const rocjitsu::Instruction *identity = nullptr; + for (unsigned repeat = 1; repeat <= 4; ++repeat) + for (auto *wf : {wf0, wf1}) + for (uint64_t mask : {~uint64_t{0}, uint64_t{0}, uint64_t{0x3333333333333333}}) { + const auto base = wf->vgpr_alloc().base; + const auto value = repeat * 1000 + base; + wf->set_exec(mask); + for (unsigned lane = 0; lane < wf->wf_size(); ++lane) { + fixture.cu()->write_vgpr(base, lane, 0xDEAD0000u + lane); + fixture.cu()->write_vgpr(base + 1, lane, value + lane); + } + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}); + ASSERT_FALSE(decoded.failed()); + if (!identity) + identity = decoded.value().get(); + EXPECT_EQ(decoded.value().get(), identity); + amdgpu::DecodedInstructionCache::ScopedReturn returned(cache, decoded.value(), kCodeBase, 7, + words); + ASSERT_TRUE(fixture.cu()->execute_instruction(decoded.value().get(), *wf).succeeded()); + EXPECT_EQ(decoded.value()->src_operand(0)->delegate(), nullptr); + for (unsigned lane = 0; lane < wf->wf_size(); ++lane) + EXPECT_EQ(fixture.cu()->read_vgpr(base, lane), + ((mask >> lane) & 1) ? value + (lane ^ 1u) : 0xDEAD0000u + lane); + } + EXPECT_EQ(decoder.calls, 1u); +} + +TEST(DecodedInstructionCacheTest, InFlightOwnershipPreventsReuse) { + CountingDecoder decoder; + amdgpu::DecodedInstructionCache cache; + std::array words{kSNop}; + std::unique_ptr in_flight; + { + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}); + ASSERT_FALSE(decoded.failed()); + amdgpu::DecodedInstructionCache::ScopedReturn returned(cache, decoded.value(), kCodeBase, 7, + words); + in_flight = std::move(decoded.value()); + } + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}); + ASSERT_FALSE(decoded.failed()); + EXPECT_NE(decoded.value().get(), in_flight.get()); + EXPECT_EQ(decoder.calls, 2u); +} + +TEST(DecodedInstructionCacheTest, FailedExecutionAndExceptionsDoNotRetainInstructions) { + CountingDecoder decoder; + amdgpu::DecodedInstructionCache cache; + std::array words{kSNop}; + { + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}); + ASSERT_FALSE(decoded.failed()); + amdgpu::DecodedInstructionCache::ScopedReturn returned(cache, decoded.value(), kCodeBase, 7, + words); + returned.discard(); + } + EXPECT_EQ(decoder.calls, 1u); + try { + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}); + ASSERT_FALSE(decoded.failed()); + amdgpu::DecodedInstructionCache::ScopedReturn returned(cache, decoded.value(), kCodeBase, 7, + words); + throw std::runtime_error("execution callback failed"); + } catch (const std::runtime_error &) { + } + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}); + ASSERT_FALSE(decoded.failed()); + EXPECT_EQ(decoder.calls, 3u); +} + +TEST(DecodedInstructionCacheTest, BorrowedVopdEncodingIsNotRetainedAcrossIssues) { + for (const auto arch : {ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_RDNA3_5, + ROCJITSU_CODE_ARCH_RDNA4, ROCJITSU_CODE_ARCH_CDNA5}) { + SCOPED_TRACE(arch); + CountingDecoder decoder(arch); + amdgpu::DecodedInstructionCache cache; + // v_dual_cndmask_b32 v2, v1, v2 :: v_dual_mov_b32 v1, 0. + std::array words{0xCA500501u, 0x02000080u}; + { + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}); + ASSERT_FALSE(decoded.failed()); + ASSERT_EQ(decoded.value()->raw_encoding(), words.data()); + amdgpu::DecodedInstructionCache::ScopedReturn returned(cache, decoded.value(), kCodeBase, 7, + words); + } + // Use the same bytes from a different fetch buffer on the next issue. + auto next_words = words; + auto decoded = cache.decode(decoder, kCodeBase, 7, next_words, {}); + ASSERT_FALSE(decoded.failed()); + EXPECT_EQ(decoded.value()->raw_encoding(), next_words.data()); + EXPECT_EQ(decoder.calls, 2u); + } +} + +TEST(DecodedInstructionCacheTest, AddressSpacesCollisionsAndClearCauseMisses) { + CountingDecoder decoder; + amdgpu::DecodedInstructionCache cache; + std::array words{kSNop}; + auto issue = [&](uint64_t pc, uint32_t vmid) { + auto decoded = cache.decode(decoder, pc, vmid, words, {}); + ASSERT_FALSE(decoded.failed()); + amdgpu::DecodedInstructionCache::ScopedReturn returned(cache, decoded.value(), pc, vmid, words); + }; + issue(kCodeBase, 7); + issue(kCodeBase, 7); + EXPECT_EQ(decoder.calls, 1u); + issue(kCodeBase, 8); + issue(kCodeBase, 7); + EXPECT_EQ(decoder.calls, 3u); + issue(kCodeBase + 4 * amdgpu::DecodedInstructionCache::kNumEntries, 7); + issue(kCodeBase, 7); + EXPECT_EQ(decoder.calls, 5u); + cache.clear(); + issue(kCodeBase, 7); + EXPECT_EQ(decoder.calls, 6u); +} + +TEST(DecodedInstructionCacheTest, DynamicStateIsNotReusedAndPoolLifetimeIsIndependent) { + amdgpu::DecodedInstructionCache cache; + std::array words{kSNop}; + { + CountingDecoder decoder; + decoder.enable_pool(); + { + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}); + ASSERT_FALSE(decoded.failed()); + amdgpu::DecodedInstructionCache::ScopedReturn returned(cache, decoded.value(), kCodeBase, 7, + words); + decoded.value()->set_data(std::make_unique()); + } + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}); + ASSERT_FALSE(decoded.failed()); + EXPECT_EQ(decoder.calls, 2u); + EXPECT_EQ(decoded.value()->data(), nullptr); + amdgpu::DecodedInstructionCache::ScopedReturn returned(cache, decoded.value(), kCodeBase, 7, + words); + } + // The decoder and its active allocator pool are gone before the cached + // instruction is destroyed. Cache entries must have independent storage. + cache.clear(); +} + +TEST(DecodedInstructionCacheTest, UncachedInstructionsAreHeapBackedWithAnActivePool) { + rocjitsu::Instruction::ScopedHeapAllocation restore_allocator; + CountingDecoder decoder; + decoder.enable_pool(); + const auto pool = static_cast(rocjitsu::Instruction::alloc_pool_); + ASSERT_NE(pool, nullptr); + amdgpu::DecodedInstructionCache cache; + std::array words{kSNop}; + for (unsigned mode = 0; mode < 4; ++mode) { + SCOPED_TRACE(mode); + std::unique_ptr in_flight; + { + const bool enabled = mode != 2; + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}, enabled); + ASSERT_FALSE(decoded.failed()); + EXPECT_FALSE(pool->owns(decoded.value().get())); + EXPECT_EQ(rocjitsu::Instruction::alloc_pool_, pool); + amdgpu::DecodedInstructionCache::ScopedReturn returned(cache, decoded.value(), kCodeBase, 7, + words, enabled); + if (mode == 0) + returned.discard(); + else if (mode == 1) + decoded.value()->set_data(std::make_unique()); + else if (mode == 3) + in_flight = std::move(decoded.value()); + } + // ArenaAlloc's non-owned-pointer fallback is the supported heap deletion + // path even when an unrelated decoder's pool is ambient on this thread. + in_flight.reset(); + EXPECT_EQ(rocjitsu::Instruction::alloc_pool_, pool); + } + EXPECT_EQ(decoder.calls, 4u); +} + +TEST(DecodedInstructionCacheTest, EntriesOutliveMultipleWorkerPools) { + amdgpu::DecodedInstructionCache cache; + std::array words{kSNop}; + const rocjitsu::Instruction *identity = nullptr; + for (unsigned worker = 0; worker < 4; ++worker) { + std::thread thread([&] { + CountingDecoder decoder; + decoder.enable_pool(); + auto decoded = cache.decode(decoder, kCodeBase, 7, words, {}); + ASSERT_FALSE(decoded.failed()); + EXPECT_FALSE(static_cast(rocjitsu::Instruction::alloc_pool_) + ->owns(decoded.value().get())); + if (!identity) + identity = decoded.value().get(); + EXPECT_EQ(decoded.value().get(), identity); + EXPECT_EQ(decoder.calls, worker == 0 ? 1u : 0u); + amdgpu::DecodedInstructionCache::ScopedReturn returned(cache, decoded.value(), kCodeBase, 7, + words); + }); + thread.join(); + } + cache.clear(); +} + // A warm instruction-cache line must not hide revocation of the VM snapshot // retained by the CU. Exercise both explicit handles and the VMID fallback. TEST(InstructionCacheCuTest, VmInvalidationRefreshesAnAlreadyCachedInstruction) {