diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/cdna5/addr_calc.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/cdna5/addr_calc.cpp index 1d82aa44c2bf..401bb84ae024 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/cdna5/addr_calc.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/cdna5/addr_calc.cpp @@ -4,7 +4,6 @@ #include "rocjitsu/isa/arch/amdgpu/cdna5/addr_calc.h" #include "rocjitsu/isa/arch/amdgpu/generated/cdna5/operand.h" #include "rocjitsu/isa/arch/amdgpu/generated/cdna5/operand_types.h" -#include "rocjitsu/isa/arch/amdgpu/shared/addr_calc_buffer.h" #include "rocjitsu/vm/amdgpu/compute_unit.h" #include "rocjitsu/vm/amdgpu/lds.h" #include "rocjitsu/vm/amdgpu/mem_state.h" @@ -21,6 +20,8 @@ namespace rocjitsu::cdna5 { namespace { +constexpr uint64_t kBufferOffsetMask = (uint64_t{1} << 45) - 1; + uint32_t scaled_vaddr_factor(const amdgpu::VectorMemState &d) { // LLVM folds gfx1250 scale_offset when the scale matches the full memory // access size. The encoded immediate offset remains a byte offset. @@ -64,18 +65,6 @@ uint32_t resolved_vgpr_base(const amdgpu::Wavefront &wf, uint32_t operand, *Isa::resolved_vgpr_offset(wf, OperandType::OPR_VGPR, operand, role); } -uint64_t buffer_base_addr(uint32_t srd0, uint32_t srd1) { - return (static_cast(srd1 & 0x01FF'FFFFu) << 32) | srd0; -} - -uint64_t buffer_num_records(uint32_t srd1, uint32_t srd2, uint32_t srd3) { - return ((static_cast(srd3 & 0x3Fu) << 32) | srd2) << 7 | ((srd1 >> 25) & 0x7Fu); -} - -uint32_t buffer_stride(uint32_t srd3) { return (srd3 >> 12) & 0xFFFFu; } - -bool buffer_oob_raw(uint32_t srd3) { return (srd3 & 0x8000'0000u) != 0; } - void init_vector_mem_state(amdgpu::Wavefront &wf, amdgpu::VectorMemState &d) { uint64_t exec = wf.exec(); d.lane_mask = exec; @@ -86,6 +75,17 @@ void init_vector_mem_state(amdgpu::Wavefront &wf, amdgpu::VectorMemState &d) { d.cu_path = wf.cu().full_path(); } +int64_t logical_buffer_offset(uint32_t index, uint32_t stride, uint32_t voffset, int32_t ioffset) { + return static_cast(index) * stride + voffset + ioffset; +} + +bool byte_range_exceeds(int64_t offset, uint32_t size, uint64_t bound) { + if (offset < 0) + return true; + uint64_t unsigned_offset = static_cast(offset); + return unsigned_offset > bound || size > bound - unsigned_offset; +} + bool decode_flat_private_address(amdgpu::Wavefront &wf, uint64_t addr, uint64_t *translated) { uint32_t lane_stride = wf.scratch_lane_size(); if (lane_stride == 0) @@ -150,6 +150,21 @@ void flat_global_calculate_addresses(const Inst &inst, amdgpu::Wavefront &wf, } // namespace +BufferResource decode_buffer_resource(uint32_t srd0, uint32_t srd1, uint32_t srd2, uint32_t srd3) { + static constexpr uint32_t stride_multipliers[] = {1, 4, 8, 32}; + BufferResource resource{}; + resource.base_address = (static_cast(srd1 & 0x01FF'FFFFu) << 32) | srd0; + resource.num_records = + ((static_cast(srd3 & 0x3Fu) << 32) | srd2) << 7 | ((srd1 >> 25) & 0x7Fu); + resource.raw_stride = (srd3 >> 12) & 0x3FFFu; + resource.stride_scale = static_cast((srd3 >> 26) & 0x3u); + resource.stride = resource.raw_stride * stride_multipliers[resource.stride_scale]; + resource.swizzle_enabled = ((srd3 >> 28) & 0x1u) != 0; + resource.oob_select = ((srd3 >> 29) & 0x1u) != 0; + resource.type = static_cast((srd3 >> 30) & 0x3u); + return resource; +} + uint64_t smem_calculate_address(const SmemMachineInst &inst, amdgpu::Wavefront &wf, uint32_t access_size_bytes) { auto &cu = wf.cu(); @@ -233,12 +248,9 @@ void mubuf_calculate_addresses(const VbufferMachineInst &inst, amdgpu::Wavefront uint32_t srd1 = amdgpu::RegisterAccess(cu).read_sgpr(sb + 1); uint32_t srd2 = amdgpu::RegisterAccess(cu).read_sgpr(sb + 2); uint32_t srd3 = amdgpu::RegisterAccess(cu).read_sgpr(sb + 3); - uint64_t base_addr = buffer_base_addr(srd0, srd1); - uint64_t num_records = buffer_num_records(srd1, srd2, srd3); - uint32_t stride = buffer_stride(srd3); - bool oob_raw = buffer_oob_raw(srd3); + BufferResource resource = decode_buffer_resource(srd0, srd1, srd2, srd3); uint32_t soffset_val = has_smem_offset(inst.soffset) ? read_sreg_m0_operand(wf, inst.soffset) : 0; - int64_t ioff = static_cast(signed_ioffset(inst.ioffset)); + int32_t ioff = signed_ioffset(inst.ioffset); uint32_t vbase = 0; if (inst.idxen || inst.offen) vbase = resolved_vgpr_base(wf, inst.vaddr, amdgpu::VgprMsbRole::Src0); @@ -248,6 +260,18 @@ void mubuf_calculate_addresses(const VbufferMachineInst &inst, amdgpu::Wavefront uint32_t reg_count = (inst.idxen && inst.offen) ? 2 : 1; vaddr_region.emplace(regs.read_vgpr_region(vbase, reg_count, exec)); } + assert(d.elem_size != 0 && d.num_elems != 0); + d.element_lane_masks.clear(); + if (ioff < 0) { + // CDNA5 requires VBUFFER IOFFSET to be non-negative. Suppress illegal + // encodings defensively so their wrapped 45-bit address cannot reach L1. + d.element_lane_masks.assign(d.num_elems, 0); + d.lane_mask = 0; + return; + } + if (!resource.swizzle_enabled) + d.element_lane_masks.assign(d.num_elems, exec); + d.lane_mask = 0; for (uint32_t lane = 0; lane < wf.wf_size(); ++lane) { if (!(exec & (1ULL << lane))) continue; @@ -261,21 +285,34 @@ void mubuf_calculate_addresses(const VbufferMachineInst &inst, amdgpu::Wavefront } else if (inst.offen) { voffset = vaddr_region->lane(0, lane); } - uint32_t offset_part = amdgpu::addr_calc::buffer_offset_part(voffset, ioff); - uint64_t total_offset = - amdgpu::addr_calc::buffer_total_offset(index, stride, offset_part, soffset_val); - bool oob; - if (!oob_raw && stride > 0) { - oob = index >= num_records; - } else { - oob = offset_part >= num_records; + int64_t logical_offset = logical_buffer_offset(index, resource.stride, voffset, ioff); + uint64_t buffer_offset = static_cast(logical_offset) & kBufferOffsetMask; + d.per_lane_addr[lane] = resource.base_address + soffset_val + buffer_offset; + + if (resource.swizzle_enabled) { + // Swizzled address generation is not modeled yet. Preserve the existing + // linear address behavior, but do not apply unswizzled bounds rules. + d.lane_mask |= uint64_t{1} << lane; + continue; } - if (oob) { - d.lane_mask &= ~(1ULL << lane); - d.per_lane_addr[lane] = 0; - } else { - d.per_lane_addr[lane] = (base_addr + total_offset) & 0xFFFFFFFFFFFFULL; + + int64_t record_offset = static_cast(soffset_val) + voffset + ioff; + int64_t total_offset = static_cast(soffset_val) + logical_offset; + for (uint32_t elem = 0; elem < d.num_elems; ++elem) { + int64_t component_offset = static_cast(elem) * d.elem_size; + bool oob = + byte_range_exceeds(total_offset + component_offset, d.elem_size, resource.num_records); + if (resource.oob_select) { + oob = oob || + byte_range_exceeds(record_offset + component_offset, d.elem_size, resource.stride); + } + if (oob) + d.element_lane_masks[elem] &= ~(uint64_t{1} << lane); + else + d.lane_mask |= uint64_t{1} << lane; } + if (!(d.lane_mask & (uint64_t{1} << lane))) + d.per_lane_addr[lane] = 0; } } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/cdna5/addr_calc.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/cdna5/addr_calc.h index 2b954116be6c..ad306b7c4c4e 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/cdna5/addr_calc.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/cdna5/addr_calc.h @@ -18,6 +18,20 @@ struct VectorMemState; namespace rocjitsu::cdna5 { +/// @brief Decoded CDNA5 buffer resource descriptor fields. +struct BufferResource { + uint64_t base_address = 0; + uint64_t num_records = 0; + uint32_t raw_stride = 0; + uint32_t stride = 0; + uint8_t stride_scale = 0; + bool swizzle_enabled = false; + bool oob_select = false; + uint8_t type = 0; +}; + +BufferResource decode_buffer_resource(uint32_t srd0, uint32_t srd1, uint32_t srd2, uint32_t srd3); + /// @brief Sign-extend a CDNA5 24-bit IOFFSET field. inline int32_t signed_ioffset(uint32_t ioffset) { return static_cast(ioffset << 8) >> 8; } diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_vector_cache.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_vector_cache.cpp index c3ef800433e7..c792104b9c21 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_vector_cache.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_vector_cache.cpp @@ -43,6 +43,23 @@ uint32_t for_each_coalesced_lane_run(const uint64_t *addrs, uint64_t lane_mask, return run_count; } +bool all_elements_use_lane_mask(std::span element_lane_masks, uint64_t lane_mask, + uint32_t num_elems) { + if (element_lane_masks.empty()) + return true; + assert(element_lane_masks.size() == num_elems); + (void)num_elems; + return std::ranges::all_of(element_lane_masks, + [lane_mask](uint64_t mask) { return mask == lane_mask; }); +} + +uint64_t fully_valid_lane_mask(std::span element_lane_masks, uint64_t lane_mask) { + uint64_t full_lane_mask = lane_mask; + for (uint64_t element_mask : element_lane_masks) + full_lane_mask &= element_mask; + return full_lane_mask; +} + } // namespace L1VectorCache::L1VectorCache(L2Cache *l2) @@ -193,10 +210,30 @@ void L1VectorCache::write_bytes(uint64_t addr, const uint8_t *src, uint32_t size void L1VectorCache::load(const uint64_t *addrs, uint64_t lane_mask, uint32_t elem_size, uint32_t num_elems, uint8_t *dst, Mtype mtype, bool non_temporal, - bool request_l1_bypass, uint32_t wf_size, uint32_t vmid) { + bool request_l1_bypass, uint32_t wf_size, uint32_t vmid, + std::span element_lane_masks) { auto coherence_guard = DeviceCacheCoherence::instance().acquire_l1_access(); synchronize_epoch_locked(); uint32_t stride = num_elems * elem_size; + if (!all_elements_use_lane_mask(element_lane_masks, lane_mask, num_elems)) { + uint64_t full_lane_mask = fully_valid_lane_mask(element_lane_masks, lane_mask); + for_each_coalesced_lane_run( + addrs, full_lane_mask, wf_size, stride, [&](uint32_t first_lane, uint32_t run_lanes) { + read_bytes(addrs[first_lane], dst + first_lane * stride, run_lanes * stride, mtype, + non_temporal, request_l1_bypass, vmid); + }); + for (uint32_t elem = 0; elem < num_elems; ++elem) { + uint64_t mask = element_lane_masks[elem] & lane_mask & ~full_lane_mask; + while (mask) { + const uint32_t lane = std::countr_zero(mask); + mask &= ~(uint64_t{1} << lane); + read_bytes(addrs[lane] + static_cast(elem) * elem_size, + dst + lane * stride + elem * elem_size, elem_size, mtype, non_temporal, + request_l1_bypass, vmid); + } + } + return; + } for_each_coalesced_lane_run( addrs, lane_mask, wf_size, stride, [&](uint32_t first_lane, uint32_t run_lanes) { read_bytes(addrs[first_lane], dst + first_lane * stride, run_lanes * stride, mtype, @@ -206,7 +243,8 @@ void L1VectorCache::load(const uint64_t *addrs, uint64_t lane_mask, uint32_t ele void L1VectorCache::store(const uint64_t *addrs, uint64_t lane_mask, uint32_t elem_size, uint32_t num_elems, const uint8_t *src, Mtype mtype, bool non_temporal, - uint32_t wf_size, uint32_t vmid) { + uint32_t wf_size, uint32_t vmid, + std::span element_lane_masks) { auto coherence_guard = DeviceCacheCoherence::instance().acquire_l1_access(); synchronize_epoch_locked(); uint32_t stride = num_elems * elem_size; @@ -214,6 +252,25 @@ void L1VectorCache::store(const uint64_t *addrs, uint64_t lane_mask, uint32_t el ++store_count_; if (active_lanes > 0) ++store_active_count_; + if (!all_elements_use_lane_mask(element_lane_masks, lane_mask, num_elems)) { + uint64_t full_lane_mask = fully_valid_lane_mask(element_lane_masks, lane_mask); + store_l2_writes_ += for_each_coalesced_lane_run( + addrs, full_lane_mask, wf_size, stride, [&](uint32_t first_lane, uint32_t run_lanes) { + write_bytes(addrs[first_lane], src + first_lane * stride, run_lanes * stride, mtype, + non_temporal, vmid); + }); + for (uint32_t elem = 0; elem < num_elems; ++elem) { + uint64_t mask = element_lane_masks[elem] & lane_mask & ~full_lane_mask; + store_l2_writes_ += std::popcount(mask); + while (mask) { + const uint32_t lane = std::countr_zero(mask); + mask &= ~(uint64_t{1} << lane); + write_bytes(addrs[lane] + static_cast(elem) * elem_size, + src + lane * stride + elem * elem_size, elem_size, mtype, non_temporal, vmid); + } + } + return; + } store_l2_writes_ += for_each_coalesced_lane_run( addrs, lane_mask, wf_size, stride, [&](uint32_t first_lane, uint32_t run_lanes) { write_bytes(addrs[first_lane], src + first_lane * stride, run_lanes * stride, mtype, diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_vector_cache.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_vector_cache.h index a13f800f560a..27e73ad56752 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_vector_cache.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/l1_vector_cache.h @@ -8,6 +8,7 @@ #include "simdojo/components/cache.h" #include +#include namespace rocjitsu { namespace amdgpu { @@ -41,13 +42,20 @@ class L1VectorCache { void set_l2(L2Cache *l2); void set_memory(GpuMemory *mem); + + /// @param element_lane_masks Empty when every element uses @p lane_mask; + /// otherwise contains exactly @p num_elems masks. In the latter form, + /// @p lane_mask is the union of lanes valid for at least one element. void load(const uint64_t *addrs, uint64_t lane_mask, uint32_t elem_size, uint32_t num_elems, uint8_t *dst, Mtype mtype, bool non_temporal, bool request_l1_bypass, uint32_t wf_size, - uint32_t vmid = 0); + uint32_t vmid = 0, std::span element_lane_masks = {}); + /// @param element_lane_masks Empty when every element uses @p lane_mask; + /// otherwise contains exactly @p num_elems masks. In the latter form, + /// @p lane_mask is the union of lanes valid for at least one element. void store(const uint64_t *addrs, uint64_t lane_mask, uint32_t elem_size, uint32_t num_elems, const uint8_t *src, Mtype mtype, bool non_temporal, uint32_t wf_size, - uint32_t vmid = 0); + uint32_t vmid = 0, std::span element_lane_masks = {}); void invalidate(uint64_t addr, uint32_t vmid = 0); void invalidate_all(); diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/mem_state.h b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/mem_state.h index 56d73e0755db..39d129765e51 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/mem_state.h +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/mem_state.h @@ -91,6 +91,10 @@ struct VectorMemState : DynamicInstState { } std::array per_lane_addr = {}; uint64_t lane_mask = 0; + /// Optional per-element lane validity. Empty means every element uses + /// lane_mask; otherwise the vector contains exactly num_elems masks and + /// lane_mask is their union. + std::vector element_lane_masks; uint64_t exec_mask = 0; ///< EXEC mask at issue time. Set by addr calc functions. ///< Writeback zeroes OOB lanes (exec_mask & ~lane_mask). uint32_t wf_size = 64; ///< Wavefront width (set from wavefront's wf_size()). diff --git a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/memory_pipeline.cpp b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/memory_pipeline.cpp index 566606290a64..39eeeb8b6505 100644 --- a/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/memory_pipeline.cpp +++ b/emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/vm/amdgpu/memory_pipeline.cpp @@ -493,12 +493,13 @@ void GlobalMemPipeline::initiate_access(Instruction &inst, Wavefront &wf) { } if (d.is_load) { - d.response_data.resize(d.wf_size * d.num_elems * d.elem_size); + d.response_data.assign(d.wf_size * d.num_elems * d.elem_size, 0); l1_->load(d.per_lane_addr.data(), d.lane_mask, d.elem_size, d.num_elems, d.response_data.data(), - d.mtype, d.non_temporal, d.request_force_l1_bypass, d.wf_size, wf.process_id()); + d.mtype, d.non_temporal, d.request_force_l1_bypass, d.wf_size, wf.process_id(), + d.element_lane_masks); } else { l1_->store(d.per_lane_addr.data(), d.lane_mask, d.elem_size, d.num_elems, d.store_data.data(), - d.mtype, d.non_temporal, d.wf_size, wf.process_id()); + d.mtype, d.non_temporal, d.wf_size, wf.process_id(), d.element_lane_masks); } } diff --git a/emulation/rocjitsu/tests/cdna5_execution_test.cpp b/emulation/rocjitsu/tests/cdna5_execution_test.cpp index a01e2ac36194..245626232b7c 100644 --- a/emulation/rocjitsu/tests/cdna5_execution_test.cpp +++ b/emulation/rocjitsu/tests/cdna5_execution_test.cpp @@ -3,6 +3,7 @@ #include "cdna5_sim_test_common.h" #include "decode_test_util.h" +#include "rocjitsu/isa/arch/amdgpu/generated/cdna5/vbuffer.h" namespace { @@ -139,6 +140,208 @@ TEST(Gfx1250ExecutionTest, Wave32VectorComparePreservesVccHiScratch) { } } +TEST(Gfx1250ExecutionTest, VbufferB128LoadsZeroAndStoresDropPartialOobDwords) { + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + + constexpr uint64_t kAddr = 0xC000; + constexpr uint32_t kResourceSgpr = 4; + constexpr uint32_t kLoadVgpr = 8; + constexpr uint32_t kStoreVgpr = 12; + constexpr std::array kInitial = {0x101u, 0x102u, 0x103u, 0x104u}; + constexpr std::array kStored = {0x201u, 0x202u, 0x203u, 0x204u}; + for (uint32_t i = 0; i < kInitial.size(); ++i) + sim.memory->write32(kAddr + i * sizeof(uint32_t), kInitial[i]); + + // gfx1250 NUM_RECORDS is a 45-bit byte count split across SRD words 1-3. + constexpr uint64_t kNumRecords = 10; + const std::array resource = { + static_cast(kAddr), + static_cast((kAddr >> 32) & 0x01FF'FFFFu) | + static_cast((kNumRecords & 0x7Fu) << 25), + static_cast(kNumRecords >> 7), + static_cast((kNumRecords >> 39) & 0x3Fu), + }; + for (uint32_t i = 0; i < resource.size(); ++i) + cu->write_sgpr(wf->sgpr_alloc().base + kResourceSgpr + i, resource[i]); + + cdna5::VbufferMachineInst machine{}; + machine.vdata = kLoadVgpr; + machine.rsrc = kResourceSgpr; + machine.soffset = cdna5::OPR_SREG_NULL; + machine.scope = 3; // System scope selects uncached accesses for direct memory checks. + + amdgpu::GlobalMemPipeline pipeline(&cu->l1_vector(), cu->l2()); + auto *load = + new cdna5::BufferLoadB128Vbuffer(reinterpret_cast(&machine)); + load->execute_impl(*wf); + pipeline.issue(load, *wf); + + EXPECT_EQ(cu->read_vgpr(wf->vgpr_alloc().base + kLoadVgpr + 0, 0), kInitial[0]); + EXPECT_EQ(cu->read_vgpr(wf->vgpr_alloc().base + kLoadVgpr + 1, 0), kInitial[1]); + EXPECT_EQ(cu->read_vgpr(wf->vgpr_alloc().base + kLoadVgpr + 2, 0), 0u); + EXPECT_EQ(cu->read_vgpr(wf->vgpr_alloc().base + kLoadVgpr + 3, 0), 0u); + + for (uint32_t i = 0; i < kStored.size(); ++i) + cu->write_vgpr(wf->vgpr_alloc().base + kStoreVgpr + i, 0, kStored[i]); + machine.vdata = kStoreVgpr; + auto *store = + new cdna5::BufferStoreB128Vbuffer(reinterpret_cast(&machine)); + store->execute_impl(*wf); + pipeline.issue(store, *wf); + + EXPECT_EQ(sim.memory->read32(kAddr), kStored[0]); + EXPECT_EQ(sim.memory->read32(kAddr + 4), kStored[1]); + EXPECT_EQ(sim.memory->read32(kAddr + 8), kInitial[2]); + EXPECT_EQ(sim.memory->read32(kAddr + 12), kInitial[3]); +} + +TEST(Gfx1250ExecutionTest, VbufferB64B96MixedLanesHonorStructuredPartialOob) { + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(0x3u); + + constexpr uint64_t kAddr = 0xC080; + constexpr uint32_t kResourceSgpr = 4; + constexpr uint32_t kSoffsetSgpr = 12; + constexpr uint32_t kAddressVgpr = 4; + constexpr uint32_t kB96LoadVgpr = 8; + constexpr uint32_t kB64LoadVgpr = 12; + constexpr uint32_t kStoreVgpr = 16; + constexpr uint64_t kNumRecords = 32; + constexpr uint32_t kRawStride = 4; + constexpr uint32_t kStrideScale = 1; + constexpr uint32_t kStride = kRawStride * 4; + const std::array resource = { + static_cast(kAddr), + static_cast((kAddr >> 32) & 0x01FF'FFFFu) | + static_cast((kNumRecords & 0x7Fu) << 25), + static_cast(kNumRecords >> 7), + static_cast((kNumRecords >> 39) & 0x3Fu) | (kRawStride << 12) | + (kStrideScale << 26) | (1u << 29), + }; + for (uint32_t i = 0; i < resource.size(); ++i) + cu->write_sgpr(wf->sgpr_alloc().base + kResourceSgpr + i, resource[i]); + cu->write_sgpr(wf->sgpr_alloc().base + kSoffsetSgpr, 4); + + const uint32_t vbase = wf->vgpr_alloc().base; + cu->write_vgpr(vbase + kAddressVgpr, 0, 0); + cu->write_vgpr(vbase + kAddressVgpr + 1, 0, 0); + cu->write_vgpr(vbase + kAddressVgpr, 1, 1); + cu->write_vgpr(vbase + kAddressVgpr + 1, 1, 8); + + constexpr std::array kLane0Initial = {0x101u, 0x102u, 0x103u}; + constexpr std::array kLane1Initial = {0x201u, 0x202u, 0x203u}; + for (uint32_t i = 0; i < kLane0Initial.size(); ++i) + sim.memory->write32(kAddr + 4 + i * sizeof(uint32_t), kLane0Initial[i]); + for (uint32_t i = 0; i < kLane1Initial.size(); ++i) + sim.memory->write32(kAddr + 28 + i * sizeof(uint32_t), kLane1Initial[i]); + + cdna5::VbufferMachineInst machine{}; + machine.rsrc = kResourceSgpr; + machine.soffset = kSoffsetSgpr; + machine.vaddr = kAddressVgpr; + machine.idxen = 1; + machine.offen = 1; + machine.scope = 3; + amdgpu::GlobalMemPipeline pipeline(&cu->l1_vector(), cu->l2()); + + machine.vdata = kB64LoadVgpr; + auto *b64_load = + new cdna5::BufferLoadB64Vbuffer(reinterpret_cast(&machine)); + b64_load->execute_impl(*wf); + pipeline.issue(b64_load, *wf); + EXPECT_EQ(cu->read_vgpr(vbase + kB64LoadVgpr, 0), kLane0Initial[0]); + EXPECT_EQ(cu->read_vgpr(vbase + kB64LoadVgpr + 1, 0), kLane0Initial[1]); + EXPECT_EQ(cu->read_vgpr(vbase + kB64LoadVgpr, 1), kLane1Initial[0]); + EXPECT_EQ(cu->read_vgpr(vbase + kB64LoadVgpr + 1, 1), 0u); + + machine.vdata = kB96LoadVgpr; + auto *b96_load = + new cdna5::BufferLoadB96Vbuffer(reinterpret_cast(&machine)); + b96_load->execute_impl(*wf); + pipeline.issue(b96_load, *wf); + for (uint32_t i = 0; i < kLane0Initial.size(); ++i) + EXPECT_EQ(cu->read_vgpr(vbase + kB96LoadVgpr + i, 0), kLane0Initial[i]); + EXPECT_EQ(cu->read_vgpr(vbase + kB96LoadVgpr, 1), kLane1Initial[0]); + EXPECT_EQ(cu->read_vgpr(vbase + kB96LoadVgpr + 1, 1), 0u); + EXPECT_EQ(cu->read_vgpr(vbase + kB96LoadVgpr + 2, 1), 0u); + + constexpr std::array kLane0Stored = {0x301u, 0x302u, 0x303u}; + constexpr std::array kLane1Stored = {0x401u, 0x402u, 0x403u}; + for (uint32_t i = 0; i < kLane0Stored.size(); ++i) { + cu->write_vgpr(vbase + kStoreVgpr + i, 0, kLane0Stored[i]); + cu->write_vgpr(vbase + kStoreVgpr + i, 1, kLane1Stored[i]); + } + machine.vdata = kStoreVgpr; + auto *b96_store = + new cdna5::BufferStoreB96Vbuffer(reinterpret_cast(&machine)); + b96_store->execute_impl(*wf); + pipeline.issue(b96_store, *wf); + + for (uint32_t i = 0; i < kLane0Stored.size(); ++i) + EXPECT_EQ(sim.memory->read32(kAddr + 4 + i * sizeof(uint32_t)), kLane0Stored[i]); + EXPECT_EQ(sim.memory->read32(kAddr + kStride + 12), kLane1Stored[0]); + EXPECT_EQ(sim.memory->read32(kAddr + kStride + 16), kLane1Initial[1]); + EXPECT_EQ(sim.memory->read32(kAddr + kStride + 20), kLane1Initial[2]); +} + +TEST(Gfx1250ExecutionTest, NonReturningVbufferB64AtomicHonorsWholePayloadBounds) { + Gfx1250Sim sim; + auto *cu = sim.cu(); + auto *wf = cu->dispatch_wf(0, 0, kGfx1250ScalarSlots, 32); + ASSERT_NE(wf, nullptr); + wf->set_exec(1u); + + constexpr uint64_t kAddr = 0xC100; + constexpr uint32_t kResourceSgpr = 4; + constexpr uint32_t kDataVgpr = 8; + constexpr uint64_t kInitial = 0x0102'0304'0506'0708ULL; + constexpr uint64_t kAddend = 0x1011'1213'1415'1617ULL; + auto write_resource = [&](uint64_t num_records) { + const std::array resource = { + static_cast(kAddr), + static_cast((kAddr >> 32) & 0x01FF'FFFFu) | + static_cast((num_records & 0x7Fu) << 25), + static_cast(num_records >> 7), + static_cast((num_records >> 39) & 0x3Fu), + }; + for (uint32_t i = 0; i < resource.size(); ++i) + cu->write_sgpr(wf->sgpr_alloc().base + kResourceSgpr + i, resource[i]); + }; + + cu->write_vgpr(wf->vgpr_alloc().base + kDataVgpr, 0, static_cast(kAddend)); + cu->write_vgpr(wf->vgpr_alloc().base + kDataVgpr + 1, 0, static_cast(kAddend >> 32)); + cdna5::VbufferMachineInst machine{}; + machine.vdata = kDataVgpr; + machine.rsrc = kResourceSgpr; + machine.soffset = cdna5::OPR_SREG_NULL; + machine.scope = 3; + machine.th = 0; + amdgpu::GlobalMemPipeline pipeline(&cu->l1_vector(), cu->l2()); + + sim.memory->write64(kAddr, kInitial); + write_resource(/*num_records=*/8); + auto *exact_end = + new cdna5::BufferAtomicAddU64Vbuffer(reinterpret_cast(&machine)); + exact_end->execute_impl(*wf); + pipeline.issue(exact_end, *wf); + EXPECT_EQ(sim.memory->read64(kAddr), kInitial + kAddend); + + sim.memory->write64(kAddr, kInitial); + write_resource(/*num_records=*/6); + auto *partial_oob = + new cdna5::BufferAtomicAddU64Vbuffer(reinterpret_cast(&machine)); + partial_oob->execute_impl(*wf); + pipeline.issue(partial_oob, *wf); + EXPECT_EQ(sim.memory->read64(kAddr), kInitial); +} + TEST(Gfx1250ExecutionTest, DivScaleWritesExplicitSdstMask) { Gfx1250Sim sim; auto *cu = sim.cu(); diff --git a/emulation/rocjitsu/tests/shared_infra_test.cpp b/emulation/rocjitsu/tests/shared_infra_test.cpp index 283f2ef74f02..7e692d37c5c5 100644 --- a/emulation/rocjitsu/tests/shared_infra_test.cpp +++ b/emulation/rocjitsu/tests/shared_infra_test.cpp @@ -1763,6 +1763,114 @@ TEST(L1VectorCacheTest, UcDwordx4RoundTripPreservesVectorTransaction) { EXPECT_EQ(std::memcmp(load_bytes.data() + sizeof(kLane0), kLane1.data(), sizeof(kLane1)), 0); } +TEST(L1VectorCacheTest, PartialElementMasksSkipMaskedLoads) { + amdgpu::GpuMemory mem("test_mem"); + amdgpu::L2Cache l2("test_l2"); + amdgpu::L1VectorCache l1(&l2); + l2.set_backing_memory(&mem); + + constexpr uint64_t kAddr = 0x7880; + constexpr std::array kLane0 = {0x101u, 0x102u, 0x103u, 0x104u}; + constexpr std::array kLane1 = {0x201u, 0x202u, 0x203u, 0x204u}; + for (uint32_t elem = 0; elem < 4; ++elem) { + mem.write32(kAddr + elem * sizeof(uint32_t), kLane0[elem]); + mem.write32(kAddr + 0x40 + elem * sizeof(uint32_t), kLane1[elem]); + } + + uint64_t addrs[64] = {}; + addrs[0] = kAddr; + addrs[1] = kAddr + 0x40; + constexpr std::array kElementMasks = {0x3, 0x1, 0x1, 0x0}; + std::array bytes{}; + l1.load(addrs, /*lane_mask=*/0x3, /*elem_size=*/4, /*num_elems=*/4, bytes.data(), + amdgpu::Mtype::UC, /*non_temporal=*/false, /*request_l1_bypass=*/false, + cdna3::Isa::WF_SIZE, /*vmid=*/0, kElementMasks); + + std::array lane0{}; + std::array lane1{}; + std::memcpy(lane0.data(), bytes.data(), sizeof(lane0)); + std::memcpy(lane1.data(), bytes.data() + sizeof(lane0), sizeof(lane1)); + EXPECT_EQ(lane0, (std::array{0x101u, 0x102u, 0x103u, 0u})); + EXPECT_EQ(lane1, (std::array{0x201u, 0u, 0u, 0u})); +} + +TEST(L1VectorCacheTest, PartialElementMasksDropSkippedStores) { + amdgpu::GpuMemory mem("test_mem"); + amdgpu::L2Cache l2("test_l2"); + amdgpu::L1VectorCache l1(&l2); + l2.set_backing_memory(&mem); + + constexpr uint64_t kAddr = 0x78c0; + constexpr std::array kInitial = {0xA0u, 0xA1u, 0xA2u, 0xA3u}; + constexpr std::array kStored = {0xB0u, 0xB1u, 0xB2u, 0xB3u}; + for (uint32_t elem = 0; elem < 4; ++elem) + mem.write32(kAddr + elem * sizeof(uint32_t), kInitial[elem]); + + uint64_t addrs[64] = {}; + addrs[0] = kAddr; + std::array bytes{}; + std::memcpy(bytes.data(), kStored.data(), sizeof(kStored)); + constexpr std::array kElementMasks = {0x1, 0x1, 0x0, 0x0}; + l1.store(addrs, /*lane_mask=*/0x1, /*elem_size=*/4, /*num_elems=*/4, bytes.data(), + amdgpu::Mtype::UC, /*non_temporal=*/false, cdna3::Isa::WF_SIZE, /*vmid=*/0, + kElementMasks); + + EXPECT_EQ(mem.read32(kAddr), kStored[0]); + EXPECT_EQ(mem.read32(kAddr + 4), kStored[1]); + EXPECT_EQ(mem.read32(kAddr + 8), kInitial[2]); + EXPECT_EQ(mem.read32(kAddr + 12), kInitial[3]); +} + +TEST(L1VectorCacheTest, PartialElementMasksCoalesceFullyValidLanes) { + amdgpu::GpuMemory mem("test_mem"); + amdgpu::L2Cache l2("test_l2"); + amdgpu::L1VectorCache l1(&l2); + l2.set_backing_memory(&mem); + + constexpr uint64_t kAddr = 0x78e0; + constexpr std::array kLane0 = {0x101u, 0x102u, 0x103u, 0x104u}; + constexpr std::array kLane1 = {0x201u, 0x202u, 0x203u, 0x204u}; + constexpr std::array kLane2 = {0x301u, 0x302u, 0x303u, 0x304u}; + uint64_t addrs[64] = {}; + addrs[0] = kAddr; + addrs[1] = kAddr + sizeof(kLane0); + addrs[2] = kAddr + 2 * sizeof(kLane0); + std::array store_bytes{}; + std::memcpy(store_bytes.data(), kLane0.data(), sizeof(kLane0)); + std::memcpy(store_bytes.data() + sizeof(kLane0), kLane1.data(), sizeof(kLane1)); + std::memcpy(store_bytes.data() + 2 * sizeof(kLane0), kLane2.data(), sizeof(kLane2)); + constexpr std::array kElementMasks = {0x7, 0x7, 0x3, 0x3}; + + l1.store(addrs, /*lane_mask=*/0x7, /*elem_size=*/4, /*num_elems=*/4, store_bytes.data(), + amdgpu::Mtype::UC, /*non_temporal=*/false, cdna3::Isa::WF_SIZE, /*vmid=*/0, + kElementMasks); + EXPECT_EQ(l1.store_l2_writes(), 3u); + EXPECT_EQ(l2.backing_write_transactions(), 3u); + EXPECT_EQ(mem.read32(kAddr + 0), kLane0[0]); + EXPECT_EQ(mem.read32(kAddr + 4), kLane0[1]); + EXPECT_EQ(mem.read32(kAddr + 8), kLane0[2]); + EXPECT_EQ(mem.read32(kAddr + 12), kLane0[3]); + EXPECT_EQ(mem.read32(kAddr + 16), kLane1[0]); + EXPECT_EQ(mem.read32(kAddr + 20), kLane1[1]); + EXPECT_EQ(mem.read32(kAddr + 24), kLane1[2]); + EXPECT_EQ(mem.read32(kAddr + 28), kLane1[3]); + EXPECT_EQ(mem.read32(kAddr + 32), kLane2[0]); + EXPECT_EQ(mem.read32(kAddr + 36), kLane2[1]); + EXPECT_EQ(mem.read32(kAddr + 40), 0u); + EXPECT_EQ(mem.read32(kAddr + 44), 0u); + + std::array load_bytes{}; + l1.load(addrs, /*lane_mask=*/0x7, /*elem_size=*/4, /*num_elems=*/4, load_bytes.data(), + amdgpu::Mtype::UC, /*non_temporal=*/false, /*request_l1_bypass=*/false, + cdna3::Isa::WF_SIZE, /*vmid=*/0, kElementMasks); + EXPECT_EQ(l2.backing_read_transactions(), 3u); + EXPECT_EQ(std::memcmp(load_bytes.data(), kLane0.data(), sizeof(kLane0)), 0); + EXPECT_EQ(std::memcmp(load_bytes.data() + sizeof(kLane0), kLane1.data(), sizeof(kLane1)), 0); + std::array loaded_lane2{}; + std::memcpy(loaded_lane2.data(), load_bytes.data() + 2 * sizeof(kLane0), sizeof(loaded_lane2)); + EXPECT_EQ(loaded_lane2, (std::array{kLane2[0], kLane2[1], 0u, 0u})); +} + TEST(L1VectorCacheTest, UcDwordx4CoalescesAdjacentLanes) { amdgpu::GpuMemory mem("test_mem"); amdgpu::L2Cache l2("test_l2"); @@ -4225,7 +4333,49 @@ TEST(RdnaAddrCalcTest, Rdna4VbufferWrapsOffsetPartBeforeBaseAddition) { EXPECT_EQ(d.per_lane_addr[0], kBase); } -TEST(RdnaAddrCalcTest, Gfx1250VbufferWrapsOffsetPartBeforeBoundsCheck) { +std::array encode_gfx1250_buffer_resource(uint64_t base, uint64_t num_records, + uint32_t stride = 0, + uint32_t stride_scale = 0, + bool swizzle = false, + bool oob_select = false, uint32_t type = 0) { + return { + static_cast(base), + static_cast((base >> 32) & 0x01FF'FFFFu) | + static_cast((num_records & 0x7Fu) << 25), + static_cast(num_records >> 7), + static_cast((num_records >> 39) & 0x3Fu) | ((stride & 0x3FFFu) << 12) | + ((stride_scale & 0x3u) << 26) | (static_cast(swizzle) << 28) | + (static_cast(oob_select) << 29) | ((type & 0x3u) << 30), + }; +} + +void write_gfx1250_buffer_resource(amdgpu::ComputeUnitCore &cu, amdgpu::Wavefront &wf, + uint32_t first_sgpr, const std::array &resource) { + for (uint32_t i = 0; i < resource.size(); ++i) + cu.write_sgpr(wf.sgpr_alloc().base + first_sgpr + i, resource[i]); +} + +TEST(Gfx1250AddrCalcTest, DecodesBufferResourceFields) { + constexpr uint64_t kBase = 0x0101'2345'6789'ABCDULL; + constexpr uint64_t kNumRecords = 0x0123'4567'89ABULL; + constexpr uint32_t kRawStride = 0x1234; + auto words = encode_gfx1250_buffer_resource(kBase, kNumRecords, kRawStride, + /*stride_scale=*/3, /*swizzle=*/true, + /*oob_select=*/true, /*type=*/2); + + cdna5::BufferResource resource = + cdna5::decode_buffer_resource(words[0], words[1], words[2], words[3]); + EXPECT_EQ(resource.base_address, kBase); + EXPECT_EQ(resource.num_records, kNumRecords); + EXPECT_EQ(resource.raw_stride, kRawStride); + EXPECT_EQ(resource.stride_scale, 3); + EXPECT_EQ(resource.stride, kRawStride * 32); + EXPECT_TRUE(resource.swizzle_enabled); + EXPECT_TRUE(resource.oob_select); + EXPECT_EQ(resource.type, 2); +} + +TEST(Gfx1250AddrCalcTest, VbufferUses45BitBufferOffsetAnd57BitBase) { amdgpu::GpuMemory mem("gfx1250_vbuffer_wrap_mem"); amdgpu::L2Cache l2("gfx1250_vbuffer_wrap_l2"); amdgpu::ComputeUnitCore::Config cfg{}; @@ -4241,13 +4391,10 @@ TEST(RdnaAddrCalcTest, Gfx1250VbufferWrapsOffsetPartBeforeBoundsCheck) { ASSERT_NE(wf, nullptr); wf->set_exec(1ULL); - constexpr uint64_t kBase = 0x2'0000'1000ULL; - uint32_t sbase = wf->sgpr_alloc().base; + constexpr uint64_t kBase = 0x0010'0002'0000'1000ULL; + constexpr uint64_t kOffset = uint64_t{1} << 32; uint32_t vbase = wf->vgpr_alloc().base; - cu->write_sgpr(sbase + 4, static_cast(kBase)); - cu->write_sgpr(sbase + 5, static_cast(kBase >> 32)); - cu->write_sgpr(sbase + 6, 0x100); - cu->write_sgpr(sbase + 7, 0); + write_gfx1250_buffer_resource(*cu, *wf, 4, encode_gfx1250_buffer_resource(kBase, kOffset + 4)); cu->write_vgpr(vbase + 4, 0, 0xFFFF'8200u); cdna5::VbufferMachineInst inst{}; @@ -4259,9 +4406,105 @@ TEST(RdnaAddrCalcTest, Gfx1250VbufferWrapsOffsetPartBeforeBoundsCheck) { inst.ioffset = 0x7E00; amdgpu::VectorMemState d(amdgpu::GLOBAL_MEM); + d.elem_size = 4; + d.num_elems = 1; cdna5::mubuf_calculate_addresses(inst, *wf, d); EXPECT_EQ(d.lane_mask, 1ULL); + EXPECT_EQ(d.per_lane_addr[0], kBase + kOffset); + EXPECT_EQ(d.element_lane_masks, (std::vector{1ULL})); +} + +TEST(Gfx1250AddrCalcTest, VbufferRangeCheckDoesNotWrapAt45Bits) { + amdgpu::GpuMemory mem("gfx1250_vbuffer_range_wrap_mem"); + amdgpu::L2Cache l2("gfx1250_vbuffer_range_wrap_l2"); + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = ROCJITSU_CODE_ARCH_CDNA5; + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = 128; + cfg.vgprs_per_wf = 16; + cfg.lds_size_kb = 64; + auto cu = amdgpu::ComputeUnitCore::create("gfx1250_vbuffer_range_wrap_cu", cfg, &mem, &l2); + ASSERT_NE(cu, nullptr); + + auto *wf = cu->dispatch_wf(0, 0, 128, 16); + ASSERT_NE(wf, nullptr); + wf->set_exec(0x1FULL); + + constexpr uint64_t kBase = 0x2'0000'1000ULL; + constexpr uint64_t kMaxNumRecords = (uint64_t{1} << 45) - 1; + constexpr uint32_t kRawStride = 8192; + write_gfx1250_buffer_resource( + *cu, *wf, 40, + encode_gfx1250_buffer_resource(kBase, kMaxNumRecords, kRawStride, /*stride_scale=*/1)); + uint32_t vbase = wf->vgpr_alloc().base; + cu->write_vgpr(vbase + 4, 0, 0); + cu->write_vgpr(vbase + 5, 0, 0); + cu->write_vgpr(vbase + 4, 1, uint32_t{1} << 30); + cu->write_vgpr(vbase + 5, 1, 0); + cu->write_vgpr(vbase + 4, 2, (uint32_t{1} << 30) - 1); + cu->write_vgpr(vbase + 5, 2, 32763); + cu->write_vgpr(vbase + 4, 3, (uint32_t{1} << 30) - 1); + cu->write_vgpr(vbase + 5, 3, 32764); + cu->write_vgpr(vbase + 4, 4, uint32_t{1} << 30); + cu->write_vgpr(vbase + 5, 4, 1); + + cdna5::VbufferMachineInst inst{}; + inst.rsrc = 40; + inst.soffset = cdna5::OPR_SREG_NULL; + inst.idxen = 1; + inst.offen = 1; + inst.vaddr = 4; + + amdgpu::VectorMemState d(amdgpu::GLOBAL_MEM); + d.elem_size = 4; + d.num_elems = 1; + cdna5::mubuf_calculate_addresses(inst, *wf, d); + EXPECT_EQ(d.lane_mask, 0x5ULL); + EXPECT_EQ(d.element_lane_masks, (std::vector{0x5ULL})); EXPECT_EQ(d.per_lane_addr[0], kBase); + EXPECT_EQ(d.per_lane_addr[1], 0ULL); + EXPECT_EQ(d.per_lane_addr[2], kBase + kMaxNumRecords - 4); + EXPECT_EQ(d.per_lane_addr[3], 0ULL); + EXPECT_EQ(d.per_lane_addr[4], 0ULL); +} + +TEST(Gfx1250AddrCalcTest, VbufferRangeCheckAcceptsExactEndOnly) { + amdgpu::GpuMemory mem("gfx1250_vbuffer_exact_end_mem"); + amdgpu::L2Cache l2("gfx1250_vbuffer_exact_end_l2"); + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = ROCJITSU_CODE_ARCH_CDNA5; + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = 128; + cfg.vgprs_per_wf = 16; + cfg.lds_size_kb = 64; + auto cu = amdgpu::ComputeUnitCore::create("gfx1250_vbuffer_exact_end_cu", cfg, &mem, &l2); + ASSERT_NE(cu, nullptr); + + auto *wf = cu->dispatch_wf(0, 0, 128, 16); + ASSERT_NE(wf, nullptr); + wf->set_exec(0x3ULL); + + constexpr uint64_t kBase = 0x2'0000'1800ULL; + write_gfx1250_buffer_resource(*cu, *wf, 40, + encode_gfx1250_buffer_resource(kBase, /*num_records=*/8)); + uint32_t vbase = wf->vgpr_alloc().base; + cu->write_vgpr(vbase + 4, 0, 4); + cu->write_vgpr(vbase + 4, 1, 5); + + cdna5::VbufferMachineInst inst{}; + inst.rsrc = 40; + inst.soffset = cdna5::OPR_SREG_NULL; + inst.offen = 1; + inst.vaddr = 4; + + amdgpu::VectorMemState d(amdgpu::GLOBAL_MEM); + d.elem_size = 4; + d.num_elems = 1; + cdna5::mubuf_calculate_addresses(inst, *wf, d); + EXPECT_EQ(d.lane_mask, 0x1ULL); + EXPECT_EQ(d.element_lane_masks, (std::vector{0x1ULL})); + EXPECT_EQ(d.per_lane_addr[0], kBase + 4); + EXPECT_EQ(d.per_lane_addr[1], 0ULL); } TEST(Gfx1250AddrCalcTest, VbufferZeroNumRecordsMasksAllLanes) { @@ -4280,9 +4523,9 @@ TEST(Gfx1250AddrCalcTest, VbufferZeroNumRecordsMasksAllLanes) { ASSERT_NE(wf, nullptr); wf->set_exec(0x3ULL); - // A null optional pointer is represented by an all-zero descriptor. Its - // zero-sized range makes every access out of bounds, so loads return zero - // and stores are dropped without touching address zero. + // The rocjitsu runtime represents a null optional pointer with an all-zero + // descriptor. This is a runtime convention, not a distinguished ISA + // descriptor encoding. Its zero-sized range makes every access out of bounds. uint32_t vbase = wf->vgpr_alloc().base; cu->write_vgpr(vbase + 4, 0, 0); cu->write_vgpr(vbase + 4, 1, 0x1000); @@ -4295,6 +4538,8 @@ TEST(Gfx1250AddrCalcTest, VbufferZeroNumRecordsMasksAllLanes) { inst.vaddr = 4; amdgpu::VectorMemState d(amdgpu::GLOBAL_MEM); + d.elem_size = 4; + d.num_elems = 1; cdna5::mubuf_calculate_addresses(inst, *wf, d); EXPECT_EQ(d.exec_mask, 0x3ULL); EXPECT_EQ(d.lane_mask, 0ULL); @@ -4319,16 +4564,13 @@ TEST(Gfx1250AddrCalcTest, VbufferStructuredStrideChecksIndexBounds) { wf->set_exec(0x3ULL); constexpr uint64_t kBase = 0x2'0000'1000ULL; - constexpr uint32_t kNumRecords = 2; + constexpr uint32_t kNumBytes = 32; constexpr uint32_t kStride = 16; - uint32_t sbase = wf->sgpr_alloc().base; uint32_t vbase = wf->vgpr_alloc().base; - cu->write_sgpr(sbase + 40, static_cast(kBase)); - cu->write_sgpr(sbase + 41, static_cast(kBase >> 32) | (kNumRecords << 25)); - cu->write_sgpr(sbase + 42, 0); - cu->write_sgpr(sbase + 43, kStride << 12); - cu->write_vgpr(vbase + 4, 0, kNumRecords - 1); - cu->write_vgpr(vbase + 4, 1, kNumRecords); + write_gfx1250_buffer_resource(*cu, *wf, 40, + encode_gfx1250_buffer_resource(kBase, kNumBytes, kStride)); + cu->write_vgpr(vbase + 4, 0, 1); + cu->write_vgpr(vbase + 4, 1, 2); cdna5::VbufferMachineInst inst{}; inst.rsrc = 40; @@ -4338,13 +4580,246 @@ TEST(Gfx1250AddrCalcTest, VbufferStructuredStrideChecksIndexBounds) { inst.vaddr = 4; amdgpu::VectorMemState d(amdgpu::GLOBAL_MEM); + d.elem_size = 4; + d.num_elems = 1; cdna5::mubuf_calculate_addresses(inst, *wf, d); EXPECT_EQ(d.exec_mask, 0x3ULL); EXPECT_EQ(d.lane_mask, 0x1ULL); - EXPECT_EQ(d.per_lane_addr[0], kBase + (kNumRecords - 1) * kStride); + EXPECT_EQ(d.per_lane_addr[0], kBase + kStride); EXPECT_EQ(d.per_lane_addr[1], 0ULL); } +TEST(Gfx1250AddrCalcTest, VbufferChecksB64B96AndB128DwordsIndependently) { + amdgpu::GpuMemory mem("gfx1250_vbuffer_partial_mem"); + amdgpu::L2Cache l2("gfx1250_vbuffer_partial_l2"); + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = ROCJITSU_CODE_ARCH_CDNA5; + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = 128; + cfg.vgprs_per_wf = 16; + cfg.lds_size_kb = 64; + auto cu = amdgpu::ComputeUnitCore::create("gfx1250_vbuffer_partial_cu", cfg, &mem, &l2); + ASSERT_NE(cu, nullptr); + + auto *wf = cu->dispatch_wf(0, 0, 128, 16); + ASSERT_NE(wf, nullptr); + wf->set_exec(1ULL); + + constexpr uint64_t kBase = 0x2'0000'2000ULL; + cdna5::VbufferMachineInst inst{}; + inst.rsrc = 40; + inst.soffset = cdna5::OPR_SREG_NULL; + + write_gfx1250_buffer_resource(*cu, *wf, 40, + encode_gfx1250_buffer_resource(kBase, /*num_records=*/6)); + amdgpu::VectorMemState b64(amdgpu::GLOBAL_MEM); + b64.elem_size = 4; + b64.num_elems = 2; + cdna5::mubuf_calculate_addresses(inst, *wf, b64); + EXPECT_EQ(b64.lane_mask, 1ULL); + EXPECT_EQ(b64.element_lane_masks, (std::vector{1ULL, 0ULL})); + + write_gfx1250_buffer_resource(*cu, *wf, 40, + encode_gfx1250_buffer_resource(kBase, /*num_records=*/10)); + amdgpu::VectorMemState b96(amdgpu::GLOBAL_MEM); + b96.elem_size = 4; + b96.num_elems = 3; + cdna5::mubuf_calculate_addresses(inst, *wf, b96); + EXPECT_EQ(b96.lane_mask, 1ULL); + EXPECT_EQ(b96.element_lane_masks, (std::vector{1ULL, 1ULL, 0ULL})); + + amdgpu::VectorMemState b128(amdgpu::GLOBAL_MEM); + b128.elem_size = 4; + b128.num_elems = 4; + cdna5::mubuf_calculate_addresses(inst, *wf, b128); + EXPECT_EQ(b128.lane_mask, 1ULL); + EXPECT_EQ(b128.element_lane_masks, (std::vector{1ULL, 1ULL, 0ULL, 0ULL})); +} + +TEST(Gfx1250AddrCalcTest, VbufferNegativeIoffsetCannotReachL1) { + amdgpu::GpuMemory mem("gfx1250_vbuffer_negative_ioffset_mem"); + amdgpu::L2Cache l2("gfx1250_vbuffer_negative_ioffset_l2"); + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = ROCJITSU_CODE_ARCH_CDNA5; + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = 128; + cfg.vgprs_per_wf = 16; + cfg.lds_size_kb = 64; + auto cu = amdgpu::ComputeUnitCore::create("gfx1250_vbuffer_negative_ioffset_cu", cfg, &mem, &l2); + ASSERT_NE(cu, nullptr); + + auto *wf = cu->dispatch_wf(0, 0, 128, 16); + ASSERT_NE(wf, nullptr); + wf->set_exec(1ULL); + + constexpr uint64_t kBase = 0x2'0000'2400ULL; + write_gfx1250_buffer_resource(*cu, *wf, 40, + encode_gfx1250_buffer_resource(kBase, /*num_records=*/4)); + cdna5::VbufferMachineInst inst{}; + inst.rsrc = 40; + inst.soffset = cdna5::OPR_SREG_NULL; + inst.ioffset = 0x00FF'FFFCu; // -4 as a signed 24-bit VBUFFER IOFFSET. + + amdgpu::VectorMemState d(amdgpu::GLOBAL_MEM); + d.elem_size = 4; + d.num_elems = 2; + cdna5::mubuf_calculate_addresses(inst, *wf, d); + + EXPECT_EQ(d.exec_mask, 1ULL); + EXPECT_EQ(d.lane_mask, 0ULL); + EXPECT_EQ(d.element_lane_masks, (std::vector{0ULL, 0ULL})); + EXPECT_EQ(d.per_lane_addr[0], 0ULL); +} + +TEST(Gfx1250AddrCalcTest, VbufferSoffsetParticipatesInNumRecordsAndStrideBounds) { + amdgpu::GpuMemory mem("gfx1250_vbuffer_soffset_bounds_mem"); + amdgpu::L2Cache l2("gfx1250_vbuffer_soffset_bounds_l2"); + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = ROCJITSU_CODE_ARCH_CDNA5; + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = 128; + cfg.vgprs_per_wf = 16; + cfg.lds_size_kb = 64; + auto cu = amdgpu::ComputeUnitCore::create("gfx1250_vbuffer_soffset_bounds_cu", cfg, &mem, &l2); + ASSERT_NE(cu, nullptr); + + auto *wf = cu->dispatch_wf(0, 0, 128, 16); + ASSERT_NE(wf, nullptr); + wf->set_exec(1ULL); + + constexpr uint64_t kBase = 0x2'0000'2800ULL; + write_gfx1250_buffer_resource( + *cu, *wf, 40, + encode_gfx1250_buffer_resource(kBase, /*num_records=*/16, /*stride=*/16, + /*stride_scale=*/0, /*swizzle=*/false, + /*oob_select=*/true)); + + cdna5::VbufferMachineInst inst{}; + inst.rsrc = 40; + inst.soffset = 8; + + uint32_t sbase = wf->sgpr_alloc().base; + cu->write_sgpr(sbase + 8, 8); + amdgpu::VectorMemState exact_end(amdgpu::GLOBAL_MEM); + exact_end.elem_size = 4; + exact_end.num_elems = 2; + cdna5::mubuf_calculate_addresses(inst, *wf, exact_end); + EXPECT_EQ(exact_end.lane_mask, 1ULL); + EXPECT_EQ(exact_end.element_lane_masks, (std::vector{1ULL, 1ULL})); + EXPECT_EQ(exact_end.per_lane_addr[0], kBase + 8); + + cu->write_sgpr(sbase + 8, 9); + amdgpu::VectorMemState one_byte_over(amdgpu::GLOBAL_MEM); + one_byte_over.elem_size = 4; + one_byte_over.num_elems = 2; + cdna5::mubuf_calculate_addresses(inst, *wf, one_byte_over); + EXPECT_EQ(one_byte_over.lane_mask, 1ULL); + EXPECT_EQ(one_byte_over.element_lane_masks, (std::vector{1ULL, 0ULL})); + EXPECT_EQ(one_byte_over.per_lane_addr[0], kBase + 9); +} + +TEST(Gfx1250AddrCalcTest, VbufferOobSelectAlsoChecksRecordStride) { + amdgpu::GpuMemory mem("gfx1250_vbuffer_oob_select_mem"); + amdgpu::L2Cache l2("gfx1250_vbuffer_oob_select_l2"); + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = ROCJITSU_CODE_ARCH_CDNA5; + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = 128; + cfg.vgprs_per_wf = 16; + cfg.lds_size_kb = 64; + auto cu = amdgpu::ComputeUnitCore::create("gfx1250_vbuffer_oob_select_cu", cfg, &mem, &l2); + ASSERT_NE(cu, nullptr); + + auto *wf = cu->dispatch_wf(0, 0, 128, 16); + ASSERT_NE(wf, nullptr); + wf->set_exec(1ULL); + + constexpr uint64_t kBase = 0x2'0000'3000ULL; + write_gfx1250_buffer_resource( + *cu, *wf, 40, + encode_gfx1250_buffer_resource(kBase, /*num_records=*/64, /*stride=*/8, + /*stride_scale=*/0, /*swizzle=*/false, + /*oob_select=*/true)); + cu->write_vgpr(wf->vgpr_alloc().base + 4, 0, 4); + + cdna5::VbufferMachineInst inst{}; + inst.rsrc = 40; + inst.soffset = cdna5::OPR_SREG_NULL; + inst.offen = 1; + inst.vaddr = 4; + + amdgpu::VectorMemState d(amdgpu::GLOBAL_MEM); + d.elem_size = 4; + d.num_elems = 2; + cdna5::mubuf_calculate_addresses(inst, *wf, d); + EXPECT_EQ(d.lane_mask, 1ULL); + EXPECT_EQ(d.element_lane_masks, (std::vector{1ULL, 0ULL})); +} + +TEST(Gfx1250AddrCalcTest, VbufferSwizzleDefersBoundsWithNoElementMasks) { + amdgpu::GpuMemory mem("gfx1250_vbuffer_swizzle_mem"); + amdgpu::L2Cache l2("gfx1250_vbuffer_swizzle_l2"); + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = ROCJITSU_CODE_ARCH_CDNA5; + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = 128; + cfg.vgprs_per_wf = 16; + cfg.lds_size_kb = 64; + auto cu = amdgpu::ComputeUnitCore::create("gfx1250_vbuffer_swizzle_cu", cfg, &mem, &l2); + ASSERT_NE(cu, nullptr); + + auto *wf = cu->dispatch_wf(0, 0, 128, 16); + ASSERT_NE(wf, nullptr); + wf->set_exec(1ULL); + write_gfx1250_buffer_resource( + *cu, *wf, 40, + encode_gfx1250_buffer_resource(/*base=*/0x4000, /*num_records=*/0, /*stride=*/16, + /*stride_scale=*/0, /*swizzle=*/true)); + + cdna5::VbufferMachineInst inst{}; + inst.rsrc = 40; + inst.soffset = cdna5::OPR_SREG_NULL; + + amdgpu::VectorMemState d(amdgpu::GLOBAL_MEM); + d.elem_size = 4; + d.num_elems = 4; + cdna5::mubuf_calculate_addresses(inst, *wf, d); + EXPECT_EQ(d.lane_mask, 1ULL); + EXPECT_TRUE(d.element_lane_masks.empty()); +} + +TEST(Gfx1250AddrCalcTest, VbufferAtomicChecksWholePayload) { + amdgpu::GpuMemory mem("gfx1250_vbuffer_atomic_oob_mem"); + amdgpu::L2Cache l2("gfx1250_vbuffer_atomic_oob_l2"); + amdgpu::ComputeUnitCore::Config cfg{}; + cfg.arch = ROCJITSU_CODE_ARCH_CDNA5; + cfg.num_wf_slots = 1; + cfg.sgprs_per_wf = 128; + cfg.vgprs_per_wf = 16; + cfg.lds_size_kb = 64; + auto cu = amdgpu::ComputeUnitCore::create("gfx1250_vbuffer_atomic_oob_cu", cfg, &mem, &l2); + ASSERT_NE(cu, nullptr); + + auto *wf = cu->dispatch_wf(0, 0, 128, 16); + ASSERT_NE(wf, nullptr); + wf->set_exec(1ULL); + write_gfx1250_buffer_resource(*cu, *wf, 40, + encode_gfx1250_buffer_resource(/*base=*/0x5000, /*num_records=*/6)); + + cdna5::VbufferMachineInst inst{}; + inst.rsrc = 40; + inst.soffset = cdna5::OPR_SREG_NULL; + + amdgpu::VectorMemState d(amdgpu::GLOBAL_MEM); + d.elem_size = 8; + d.num_elems = 1; + d.atomic_op = amdgpu::AtomicOp::ADD; + cdna5::mubuf_calculate_addresses(inst, *wf, d); + EXPECT_EQ(d.lane_mask, 0ULL); + EXPECT_EQ(d.element_lane_masks, (std::vector{0ULL})); + EXPECT_EQ(d.per_lane_addr[0], 0ULL); +} + void expect_vector_lane_reads_use_own_wave_vgprs(rj_code_arch_t arch) { amdgpu::GpuMemory mem("rdna_lane_read_mem"); amdgpu::L2Cache l2("rdna_lane_read_l2");