Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,13 @@ inline void scalar_buffer_load_mask(ScalarMemState *state, int64_t offset, uint6

/// Scalar buffer resource layout shared by GFX9 and RDNA: a 48-bit base,
/// 14-bit stride and 32-bit record count. Stride affects bounds, not the address.
///
/// @p align_mask clears low bits of the returned address. @p size_align_mask
/// clears low bits of the byte bound before the per-DWORD mask is built; zero
/// leaves the bound unrounded, so a DWORD counts as in range when its first byte
/// is. GFX9 passes 3, so a DWORD must lie wholly in range, and RDNA4 passes its
/// access alignment. RDNA1 through RDNA3.5 keep zero; their hardware rule is
/// unverified.
inline std::optional<uint64_t> scalar_buffer_address(Wavefront &wf, uint32_t selector,
uint64_t base, int64_t offset,
ScalarMemState *state, uint64_t align_mask = 3,
Expand Down Expand Up @@ -114,8 +121,10 @@ std::optional<uint64_t> smem_calculate_address(const SmemInst &inst, amdgpu::Wav
wf.report_undefined_behavior("negative combined scalar-memory offset");
inst_offset &= ~0x3LL;
reg_offset &= kDwordMask;
// A DWORD is in range only if all four bytes are, so round the bound down.
if (gfx9_smem_is_buffer_op(inst.op))
Comment thread
accauble marked this conversation as resolved.
Resolved
return scalar_buffer_address(wf, inst.sbase * 2, *base, inst_offset + reg_offset, state);
return scalar_buffer_address(wf, inst.sbase * 2, *base, inst_offset + reg_offset, state,
/*align_mask=*/3, /*size_align_mask=*/3);
const uint64_t addr = *base + inst_offset + reg_offset;
util::Logger::vm([&](auto &os) {
static thread_local uint64_t smem_count = 0;
Expand Down
12 changes: 12 additions & 0 deletions emulation/rocjitsu/tests/scalar_buffer_store_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,18 @@ TEST_P(ScalarBufferStore, BufferStoresRespectPerDwordBounds) {
EXPECT_EQ(memory_.read32(0x400c), 10u);
}

// gfx90a: a DWORD only partly inside the buffer is not written.
TEST_P(ScalarBufferStore, BufferStoreSkipsDwordThatEndsPastTheBuffer) {
for (uint32_t records : {13u, 16u}) {
SCOPED_TRACE(records);
base(records);
memory_.write32(0x400c, 10);
put(32, 100);
ASSERT_NO_FATAL_FAILURE(issue(24, true, 12)); // S_BUFFER_STORE_DWORD at offset 12.
EXPECT_EQ(memory_.read32(0x400c), records == 16 ? 100u : 10u);
}
}

INSTANTIATE_TEST_SUITE_P(Cdna, ScalarBufferStore,
testing::Values(ROCJITSU_CODE_ARCH_CDNA1, ROCJITSU_CODE_ARCH_CDNA2,
ROCJITSU_CODE_ARCH_CDNA3, ROCJITSU_CODE_ARCH_CDNA4));
Expand Down
186 changes: 130 additions & 56 deletions emulation/rocjitsu/tests/scalar_buffer_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
#include "decode_test_util.h"
#include "rocjitsu/isa/arch/amdgpu/cdna5/addr_calc.h"
#include "rocjitsu/isa/arch/amdgpu/cdna5/isa.h"
#include "rocjitsu/isa/arch/amdgpu/generated/cdna3/builders.h"
#include "rocjitsu/isa/arch/amdgpu/generated/cdna3/opcodes.h"
#include "rocjitsu/isa/arch/amdgpu/rdna1/addr_calc.h"
#include "rocjitsu/isa/arch/amdgpu/rdna1/isa.h"
#include "rocjitsu/isa/arch/amdgpu/rdna2/addr_calc.h"
Expand Down Expand Up @@ -38,13 +40,68 @@ namespace {

using namespace rocjitsu;

bool is_gfx9(rj_code_arch_t arch) {
return arch == ROCJITSU_CODE_ARCH_CDNA1 || arch == ROCJITSU_CODE_ARCH_CDNA2 ||
arch == ROCJITSU_CODE_ARCH_CDNA3 || arch == ROCJITSU_CODE_ARCH_CDNA4;
}

// One wave executing scalar buffer instructions through the scalar pipeline, on
// physical memory or through an identity-translated address space.
class ScalarBufferHarness {
public:
~ScalarBufferHarness() {
if (wf)
wf->halt();
}

void init(rj_code_arch_t arch, bool translated) {
l2.set_backing_memory(&mem);
amdgpu::ComputeUnitCore::Config cfg{};
cfg.arch = arch;
cfg.num_wf_slots = 1;
cfg.sgprs_per_wf = is_gfx9(arch) ? 102 : 106;
cfg.vgprs_per_wf = 16;
cfg.lds_size_kb = 64;
cu = amdgpu::ComputeUnitCore::create("scalar_buffer", cfg, &mem, &l2);
decoder = Decoder::create(arch);
wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, 16);
ASSERT_NE(wf, nullptr);
if (translated) {
const auto address_space =
vm.register_address_space(0, std::make_shared<amdgpu::IdentityAddressSpaceTranslator>(),
std::make_shared<amdgpu::GpuMemoryPhysicalAccess>(mem));
ASSERT_TRUE(address_space);
cu->set_gpu_vm(&vm);
wf->set_address_space(address_space);
}
}

void write_sgpr(unsigned reg, uint32_t value) {
cu->write_sgpr(wf->sgpr_alloc().base + reg, value);
}
uint32_t read_sgpr(unsigned reg) { return cu->read_sgpr(wf->sgpr_alloc().base + reg); }

void execute(const std::array<uint32_t, 2> &words) {
std::unique_ptr<Instruction> inst(decode_valid(*decoder, words.data()));
ASSERT_NE(inst, nullptr);
ASSERT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded());
ASSERT_NE(inst->data(), nullptr);
pipeline.issue(inst.release(), *wf);
}

amdgpu::GpuMemory mem{"scalar_buffer_mem"};
amdgpu::L2Cache l2{"scalar_buffer_l2"};
amdgpu::L1ScalarCache l1{&l2};
amdgpu::ScalarMemPipeline pipeline{&l1};
std::unique_ptr<amdgpu::ComputeUnitCore> cu;
std::unique_ptr<Decoder> decoder;
amdgpu::Wavefront *wf = nullptr;
amdgpu::GpuVm vm;
};

class ScalarBufferInstructionTest : public ::testing::TestWithParam<rj_code_arch_t> {
protected:
bool gfx9() const {
const auto arch = GetParam();
return arch == ROCJITSU_CODE_ARCH_CDNA1 || arch == ROCJITSU_CODE_ARCH_CDNA2 ||
arch == ROCJITSU_CODE_ARCH_CDNA3 || arch == ROCJITSU_CODE_ARCH_CDNA4;
}
bool gfx9() const { return is_gfx9(GetParam()); }
};

INSTANTIATE_TEST_SUITE_P(AllArchitectures, ScalarBufferInstructionTest,
Expand Down Expand Up @@ -120,50 +177,27 @@ TEST_P(ScalarBufferInstructionTest, ScalarBufferLoadsZeroOnlyOutOfRangeDwords) {
SCOPED_TRACE(translated);
const auto arch = GetParam();
const bool gfx12 = arch == ROCJITSU_CODE_ARCH_RDNA4 || arch == ROCJITSU_CODE_ARCH_CDNA5;
amdgpu::GpuMemory mem("scalar_buffer_mem");
amdgpu::L2Cache l2("scalar_buffer_l2");
l2.set_backing_memory(&mem);
amdgpu::L1ScalarCache l1(&l2);
amdgpu::ScalarMemPipeline pipeline(&l1);
amdgpu::ComputeUnitCore::Config cfg{};
cfg.arch = arch;
cfg.num_wf_slots = 1;
cfg.sgprs_per_wf = gfx9() ? 102 : 106;
cfg.vgprs_per_wf = 16;
cfg.lds_size_kb = 64;
auto cu = amdgpu::ComputeUnitCore::create("scalar_buffer", cfg, &mem, &l2);
auto decoder = Decoder::create(arch);
auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, 16);
ASSERT_NE(wf, nullptr);
amdgpu::GpuVm vm;
if (translated) {
const auto address_space =
vm.register_address_space(0, std::make_shared<amdgpu::IdentityAddressSpaceTranslator>(),
std::make_shared<amdgpu::GpuMemoryPhysicalAccess>(mem));
ASSERT_TRUE(address_space);
cu->set_gpu_vm(&vm);
wf->set_address_space(address_space);
}
const auto sb = wf->sgpr_alloc().base;
ScalarBufferHarness harness;
ASSERT_NO_FATAL_FAILURE(harness.init(arch, translated));
constexpr uint64_t base = 0x1000;
for (uint32_t i = 0; i < 32; ++i)
mem.write32(base + 4 * i, 100 + i);
harness.mem.write32(base + 4 * i, 100 + i);
for (uint32_t stride : {0u, 4u}) {
const uint32_t records = stride ? 3 : 12;
cu->write_sgpr(sb, base);
harness.write_sgpr(0, base);
if (arch == ROCJITSU_CODE_ARCH_CDNA5) {
cu->write_sgpr(sb + 1, records << 25);
cu->write_sgpr(sb + 2, 0);
harness.write_sgpr(1, records << 25);
harness.write_sgpr(2, 0);
// STRIDE_SCALE must be ignored by scalar loads.
cu->write_sgpr(sb + 3, (stride << 12) | (3u << 26));
harness.write_sgpr(3, (stride << 12) | (3u << 26));
} else {
cu->write_sgpr(sb + 1, stride << 16);
cu->write_sgpr(sb + 2, records);
cu->write_sgpr(sb + 3, 0);
harness.write_sgpr(1, stride << 16);
harness.write_sgpr(2, records);
harness.write_sgpr(3, 0);
}
for (uint32_t offset : {0u, 4u, 12u, 16u}) {
for (uint32_t i = 0; i < 4; ++i)
cu->write_sgpr(sb + 8 + i, 0xdeadbeef);
harness.write_sgpr(8 + i, 0xdeadbeef);
// s_buffer_load_{dwordx4,b128} s[8:11], s[0:3], offset.
const uint32_t null_offset =
(arch == ROCJITSU_CODE_ARCH_RDNA1 || arch == ROCJITSU_CODE_ARCH_RDNA2) ? 0xfa000000u
Expand All @@ -172,36 +206,76 @@ TEST_P(ScalarBufferInstructionTest, ScalarBufferLoadsZeroOnlyOutOfRangeDwords) {
: gfx12 ? 0xf4024200u
: 0xf4280200u,
offset | (gfx9() ? 0u : null_offset)};
std::unique_ptr<Instruction> inst(decode_valid(*decoder, words.data()));
ASSERT_NE(inst, nullptr);
ASSERT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded());
ASSERT_NE(inst->data(), nullptr);
pipeline.issue(inst.release(), *wf);
ASSERT_NO_FATAL_FAILURE(harness.execute(words));
for (uint32_t i = 0; i < 4; ++i)
EXPECT_EQ(cu->read_sgpr(sb + 8 + i), offset + i * 4 < 12 ? 100 + offset / 4 + i : 0)
EXPECT_EQ(harness.read_sgpr(8 + i), offset + i * 4 < 12 ? 100 + offset / 4 + i : 0)
<< "offset=" << offset << " stride=" << stride << " dword=" << i;
}
}
// Fully out-of-bounds loads must not touch even an unmapped base address.
cu->write_sgpr(sb, 0);
cu->write_sgpr(sb + 1, 0);
cu->write_sgpr(sb + 2, 0);
cu->write_sgpr(sb + 3, 0);
for (uint32_t i = 0; i < 4; ++i)
harness.write_sgpr(i, 0);
const uint32_t null_offset =
(arch == ROCJITSU_CODE_ARCH_RDNA1 || arch == ROCJITSU_CODE_ARCH_RDNA2) ? 0xfa000000u
: 0xf8000000u;
const std::array<uint32_t, 2> words{gfx9() ? 0xc02a0200u
: gfx12 ? 0xf4024200u
: 0xf4280200u,
gfx9() ? 0u : null_offset};
std::unique_ptr<Instruction> inst(decode_valid(*decoder, words.data()));
ASSERT_NE(inst, nullptr);
ASSERT_TRUE(cu->execute_instruction(inst.get(), *wf).succeeded());
ASSERT_NE(inst->data(), nullptr);
pipeline.issue(inst.release(), *wf);
ASSERT_NO_FATAL_FAILURE(harness.execute(words));
for (uint32_t i = 0; i < 4; ++i)
EXPECT_EQ(cu->read_sgpr(sb + 8 + i), 0u);
wf->halt();
EXPECT_EQ(harness.read_sgpr(8 + i), 0u);
}
}

// The cases are gfx90a hardware results: a DWORD only partly inside the buffer
// reads zero.
TEST_P(ScalarBufferInstructionTest, Gfx9LoadsZeroDwordsThatEndPastTheBuffer) {
if (!gfx9())
GTEST_SKIP() << "GFX9 scalar buffer bounds";
struct Case {
uint16_t opcode;
uint32_t offset;
uint32_t stride;
uint32_t records;
std::array<uint32_t, 2> expected;
};
constexpr std::array cases{
Case{cdna3::kSBufferLoadDwordSmem, 12, 0, 12, {0, 0}},
Case{cdna3::kSBufferLoadDwordSmem, 12, 0, 13, {0, 0}},
Case{cdna3::kSBufferLoadDwordSmem, 12, 0, 15, {0, 0}},
Case{cdna3::kSBufferLoadDwordSmem, 12, 0, 16, {103, 0}},
// Stride 5 with 3 records gives a 15-byte bound.
Comment thread
accauble marked this conversation as resolved.
Resolved
Case{cdna3::kSBufferLoadDwordSmem, 4, 5, 3, {101, 0}},
Case{cdna3::kSBufferLoadDwordSmem, 12, 5, 3, {0, 0}},
Case{cdna3::kSBufferLoadDwordx2Smem, 16, 0, 17, {0, 0}},
Case{cdna3::kSBufferLoadDwordx2Smem, 16, 0, 20, {104, 0}},
Case{cdna3::kSBufferLoadDwordx2Smem, 16, 0, 23, {104, 0}},
Case{cdna3::kSBufferLoadDwordx2Smem, 16, 0, 24, {104, 105}},
};
for (bool translated : {false, true}) {
SCOPED_TRACE(translated);
ScalarBufferHarness harness;
ASSERT_NO_FATAL_FAILURE(harness.init(GetParam(), translated));
Comment thread
accauble marked this conversation as resolved.
Resolved
constexpr uint64_t base = 0x1000;
for (uint32_t i = 0; i < 8; ++i)
harness.mem.write32(base + 4 * i, 100 + i);
harness.write_sgpr(0, base);
harness.write_sgpr(3, 0);
for (const auto &test_case : cases) {
SCOPED_TRACE(testing::Message()
<< "op=" << test_case.opcode << " offset=" << test_case.offset
<< " stride=" << test_case.stride << " records=" << test_case.records);
harness.write_sgpr(1, test_case.stride << 16);
harness.write_sgpr(2, test_case.records);
harness.write_sgpr(8, 0xdeadbeef);
harness.write_sgpr(9, 0xdeadbeef);
ASSERT_NO_FATAL_FAILURE(harness.execute(
cdna3::build_smem(test_case.opcode, {.sdata = 8, .imm = 1, .offset = test_case.offset})));
const bool two_dwords = test_case.opcode == cdna3::kSBufferLoadDwordx2Smem;
EXPECT_EQ(harness.read_sgpr(8), test_case.expected[0]);
EXPECT_EQ(harness.read_sgpr(9), two_dwords ? test_case.expected[1] : 0xdeadbeefu);
}
}
}

Expand Down
Loading