Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@
/// The encoding format of a family is stable across generations, but the opcodes
/// are not. For example, s_branch is opcode 2 on GFX9 (CDNA1-4) and 32 on GFX12
/// (RDNA4). Every build_* therefore takes rj_code_arch_t. The s_delay_alu builder
/// returns failure for an unsupported arch; other builders currently throw
/// util::UnimplementedInst. Scalar-operand codes
/// and SOPP layout builder return failure for an unsupported arch; other builders
/// currently throw util::UnimplementedInst. Scalar-operand codes
/// (VCC, EXEC, M0) and inline constants come from the generated operand tables
/// (operand_types.h); see scalar_operand_m0 / scalar_operand_vcc_lo /
/// scalar_operand_exec_lo.
Expand Down Expand Up @@ -117,8 +117,11 @@ inline constexpr uint16_t kDelayAluSaluDep1 = 9;
}

/// @brief Build a SOPP word using the generated layout for @p arch.
[[nodiscard]] inline constexpr uint32_t build_sopp_encoding(rj_code_arch_t arch, uint16_t op,
uint16_t simm16) {
/// @returns Failure for a non-AMDGPU architecture.
/// @param error Optional caller-owned diagnostic emitter, used only on failure.
inline constexpr util::FailureOr<uint32_t>
build_sopp_encoding(rj_code_arch_t arch, uint16_t op, uint16_t simm16,
const util::DiagnosticEmitter *error = nullptr) {
switch (arch) {
case ROCJITSU_CODE_ARCH_CDNA1:
return cdna1::build_sopp(op, {.simm16 = simm16})[0];
Expand All @@ -141,7 +144,8 @@ inline constexpr uint16_t kDelayAluSaluDep1 = 9;
case ROCJITSU_CODE_ARCH_CDNA5:
return cdna5::build_sopp(op, {.simm16 = simm16})[0];
default:
throw util::UnimplementedInst("SOPP builder for target architecture");
return error != nullptr ? error->emit() << "unsupported target architecture for SOPP encoding"
: util::Result::failure();
}
}

Expand Down Expand Up @@ -736,7 +740,9 @@ inline constexpr uint16_t kDelayAluSaluDep1 = 9;
/// @param arch Target ISA architecture.
/// @returns The encoded 32-bit instruction word.
[[nodiscard]] inline constexpr uint32_t build_s_branch(int16_t offset_dwords, rj_code_arch_t arch) {
return build_sopp_encoding(arch, sopp_op_branch(arch), static_cast<uint16_t>(offset_dwords));
// Opcode selection rejects architectures without a generated SOPP layout.
const uint32_t op = sopp_op_branch(arch);
return build_sopp_encoding(arch, op, static_cast<uint16_t>(offset_dwords)).value();
}

/// @brief Encode an s_getpc_b64 instruction for the given target ISA.
Expand Down Expand Up @@ -798,22 +804,28 @@ inline constexpr uint16_t kDelayAluSaluDep1 = 9;
/// @returns The encoded 32-bit instruction word.
[[nodiscard]] inline constexpr uint32_t
build_s_nop(uint16_t cycles = 0, rj_code_arch_t arch = ROCJITSU_CODE_ARCH_RDNA4) {
return build_sopp_encoding(arch, sopp_op_nop(arch), cycles);
// Opcode selection rejects architectures without a generated SOPP layout.
const uint32_t op = sopp_op_nop(arch);
return build_sopp_encoding(arch, op, cycles).value();
}

/// @brief Encode an s_endpgm instruction for the given target ISA.
///
/// @param arch Target ISA architecture.
/// @returns The encoded 32-bit instruction word.
[[nodiscard]] inline constexpr uint32_t build_s_endpgm(rj_code_arch_t arch) {
return build_sopp_encoding(arch, sopp_op_endpgm(arch), 0);
// Opcode selection rejects architectures without a generated SOPP layout.
const uint32_t op = sopp_op_endpgm(arch);
return build_sopp_encoding(arch, op, 0).value();
}

/// @brief Encode an s_trap instruction for the given target ISA.
///
/// @details The immediate is a caller-defined trap code, not a printable message.
[[nodiscard]] inline constexpr uint32_t build_s_trap(rj_code_arch_t arch, uint16_t simm16 = 0) {
return build_sopp_encoding(arch, sopp_op_trap(arch), simm16);
// Opcode selection rejects architectures without a generated SOPP layout.
const uint32_t op = sopp_op_trap(arch);
return build_sopp_encoding(arch, op, simm16).value();
}

/// @brief Encode s_delay_alu for the given target ISA.
Expand All @@ -838,7 +850,7 @@ build_s_delay_alu(uint16_t simm16, rj_code_arch_t arch,
[[nodiscard]] inline constexpr std::optional<uint32_t> build_s_wait_xcnt(rj_code_arch_t arch) {
if (arch != ROCJITSU_CODE_ARCH_CDNA5)
return std::nullopt;
return build_sopp_encoding(arch, cdna5::kSWaitXcntSopp, 0);
return build_sopp_encoding(arch, cdna5::kSWaitXcntSopp, 0).value();
}

/// @brief Encode s_mov_b32 for the given target ISA.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -225,25 +225,25 @@ build_wait_scalar_loads_complete(rj_code_arch_t arch,
const util::DiagnosticEmitter *error = nullptr) {
switch (arch) {
case ROCJITSU_CODE_ARCH_CDNA1:
return build_sopp_encoding(arch, cdna1::kSWaitcntSopp, 0);
return build_sopp_encoding(arch, cdna1::kSWaitcntSopp, 0, error);
case ROCJITSU_CODE_ARCH_CDNA2:
return build_sopp_encoding(arch, cdna2::kSWaitcntSopp, 0);
return build_sopp_encoding(arch, cdna2::kSWaitcntSopp, 0, error);
case ROCJITSU_CODE_ARCH_CDNA3:
return build_sopp_encoding(arch, cdna3::kSWaitcntSopp, 0);
return build_sopp_encoding(arch, cdna3::kSWaitcntSopp, 0, error);
case ROCJITSU_CODE_ARCH_CDNA4:
return build_sopp_encoding(arch, cdna4::kSWaitcntSopp, 0);
return build_sopp_encoding(arch, cdna4::kSWaitcntSopp, 0, error);
case ROCJITSU_CODE_ARCH_RDNA1:
return build_sopp_encoding(arch, rdna1::kSWaitcntSopp, 0);
return build_sopp_encoding(arch, rdna1::kSWaitcntSopp, 0, error);
case ROCJITSU_CODE_ARCH_RDNA2:
return build_sopp_encoding(arch, rdna2::kSWaitcntSopp, 0);
return build_sopp_encoding(arch, rdna2::kSWaitcntSopp, 0, error);
case ROCJITSU_CODE_ARCH_RDNA3:
return build_sopp_encoding(arch, rdna3::kSWaitcntSopp, 0);
return build_sopp_encoding(arch, rdna3::kSWaitcntSopp, 0, error);
case ROCJITSU_CODE_ARCH_RDNA3_5:
return build_sopp_encoding(arch, rdna3_5::kSWaitcntSopp, 0);
return build_sopp_encoding(arch, rdna3_5::kSWaitcntSopp, 0, error);
case ROCJITSU_CODE_ARCH_RDNA4:
return build_sopp_encoding(arch, rdna4::kSWaitKmcntSopp, 0);
return build_sopp_encoding(arch, rdna4::kSWaitKmcntSopp, 0, error);
case ROCJITSU_CODE_ARCH_CDNA5:
return build_sopp_encoding(arch, cdna5::kSWaitKmcntSopp, 0);
return build_sopp_encoding(arch, cdna5::kSWaitKmcntSopp, 0, error);
default:
if (error != nullptr)
return error->emit() << "unsupported target architecture for the scalar-load wait";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -155,11 +155,11 @@ build_v_readlane_b32(uint16_t sgpr_dst, uint16_t vgpr_src, uint16_t lane, rj_cod
[[nodiscard]] inline uint32_t build_wait_stores_complete(rj_code_arch_t arch) {
switch (arch) {
case ROCJITSU_CODE_ARCH_CDNA3:
return build_sopp_encoding(arch, cdna3::kSWaitcntSopp, 0);
return build_sopp_encoding(arch, cdna3::kSWaitcntSopp, 0).value();
case ROCJITSU_CODE_ARCH_CDNA4:
return build_sopp_encoding(arch, cdna4::kSWaitcntSopp, 0);
return build_sopp_encoding(arch, cdna4::kSWaitcntSopp, 0).value();
case ROCJITSU_CODE_ARCH_RDNA4:
return build_sopp_encoding(arch, rdna4::kSWaitStorecntSopp, 0);
return build_sopp_encoding(arch, rdna4::kSWaitStorecntSopp, 0).value();
default:
throw util::UnimplementedInst("store-completion wait for target architecture");
}
Expand All @@ -169,11 +169,11 @@ build_v_readlane_b32(uint16_t sgpr_dst, uint16_t vgpr_src, uint16_t lane, rj_cod
[[nodiscard]] inline uint32_t build_wait_loads_complete(rj_code_arch_t arch) {
switch (arch) {
case ROCJITSU_CODE_ARCH_CDNA3:
return build_sopp_encoding(arch, cdna3::kSWaitcntSopp, 0);
return build_sopp_encoding(arch, cdna3::kSWaitcntSopp, 0).value();
case ROCJITSU_CODE_ARCH_CDNA4:
return build_sopp_encoding(arch, cdna4::kSWaitcntSopp, 0);
return build_sopp_encoding(arch, cdna4::kSWaitcntSopp, 0).value();
case ROCJITSU_CODE_ARCH_RDNA4:
return build_sopp_encoding(arch, rdna4::kSWaitLoadcntSopp, 0);
return build_sopp_encoding(arch, rdna4::kSWaitLoadcntSopp, 0).value();
default:
throw util::UnimplementedInst("load-completion wait for target architecture");
}
Expand Down Expand Up @@ -201,16 +201,16 @@ build_v_readlane_b32(uint16_t sgpr_dst, uint16_t vgpr_src, uint16_t lane, rj_cod
// probe call still reaches emit_probe_call and needs the unconditional boundary
// drain. Its monolithic s_waitcnt 0 drains vmcnt+lgkmcnt, same as CDNA3/4.
case ROCJITSU_CODE_ARCH_CDNA2:
return {build_sopp_encoding(arch, cdna2::kSWaitcntSopp, 0)};
return {build_sopp_encoding(arch, cdna2::kSWaitcntSopp, 0).value()};
case ROCJITSU_CODE_ARCH_CDNA3:
return {build_sopp_encoding(arch, cdna3::kSWaitcntSopp, 0)};
return {build_sopp_encoding(arch, cdna3::kSWaitcntSopp, 0).value()};
case ROCJITSU_CODE_ARCH_CDNA4:
return {build_sopp_encoding(arch, cdna4::kSWaitcntSopp, 0)};
return {build_sopp_encoding(arch, cdna4::kSWaitcntSopp, 0).value()};
case ROCJITSU_CODE_ARCH_RDNA4:
return {build_sopp_encoding(arch, rdna4::kSWaitLoadcntDscntSopp, 0),
build_sopp_encoding(arch, rdna4::kSWaitSamplecntSopp, 0),
build_sopp_encoding(arch, rdna4::kSWaitBvhcntSopp, 0),
build_sopp_encoding(arch, rdna4::kSWaitKmcntSopp, 0)};
return {build_sopp_encoding(arch, rdna4::kSWaitLoadcntDscntSopp, 0).value(),
build_sopp_encoding(arch, rdna4::kSWaitSamplecntSopp, 0).value(),
build_sopp_encoding(arch, rdna4::kSWaitBvhcntSopp, 0).value(),
build_sopp_encoding(arch, rdna4::kSWaitKmcntSopp, 0).value()};
default:
throw util::UnimplementedInst("all-loads wait for target architecture");
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -625,8 +625,11 @@ void append_direct_branch_island_pool(std::vector<uint8_t> &kernel_text, KernelT
const std::string_view mnemonic = inst.mnemonic();
const bool invert_to_next =
mnemonic == "s_cbranch_scc0" || mnemonic == "s_cbranch_vccz" || mnemonic == "s_cbranch_execz";
return build_sopp_encoding(arch, invert_to_next ? op + 1u : op - 1u,
static_cast<uint16_t>(*skip));
const util::FailureOr<uint32_t> word =
build_sopp_encoding(arch, invert_to_next ? op + 1u : op - 1u, static_cast<uint16_t>(*skip));
if (word.failed())
return std::nullopt;
return word.value();
}

[[nodiscard]] std::optional<uint16_t> direct_call_return_sgpr(const Instruction &inst,
Expand Down
2 changes: 1 addition & 1 deletion emulation/rocjitsu/tests/dbi/log_record_probe.h
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ inline void append(std::vector<uint32_t> &words, std::span<const uint32_t> more)
// 0xfffe is the value LLVM's amdgpu-wait-sgpr-hazards pass emits for it.
inline void append_salu_sgpr_write_wait(std::vector<uint32_t> &words, rj_code_arch_t arch) {
if (arch == ROCJITSU_CODE_ARCH_RDNA4)
words.push_back(build_sopp_encoding(arch, rdna4::kSWaitAluSopp, 0xfffe));
words.push_back(build_sopp_encoding(arch, rdna4::kSWaitAluSopp, 0xfffe).value());
}

} // namespace detail
Expand Down
9 changes: 6 additions & 3 deletions emulation/rocjitsu/tests/dbi/scalar_builders.h
Original file line number Diff line number Diff line change
Expand Up @@ -105,13 +105,16 @@ static_assert(cdna3::OPR_SSRC_SRC_LITERAL == kScalarSrcLiteral &&
switch (arch) {
case ROCJITSU_CODE_ARCH_CDNA3:
return build_sopp_encoding(arch, scc1 ? cdna3::kSCbranchScc1Sopp : cdna3::kSCbranchScc0Sopp,
offset);
offset)
.value();
case ROCJITSU_CODE_ARCH_CDNA4:
return build_sopp_encoding(arch, scc1 ? cdna4::kSCbranchScc1Sopp : cdna4::kSCbranchScc0Sopp,
offset);
offset)
.value();
case ROCJITSU_CODE_ARCH_RDNA4:
return build_sopp_encoding(arch, scc1 ? rdna4::kSCbranchScc1Sopp : rdna4::kSCbranchScc0Sopp,
offset);
offset)
.value();
default:
throw util::UnimplementedInst("s_cbranch_scc for target architecture");
}
Expand Down
39 changes: 38 additions & 1 deletion emulation/rocjitsu/tests/patch/instruction_builder_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,40 @@ TEST(InstructionBuilder, Sop2SetsEncodingPrefix) {
EXPECT_EQ((word >> 30) & 0x3u, 0x2u);
}

TEST(InstructionBuilder, SoppEncodingReturnsEncodingOrFailure) {
constexpr auto invalid = build_sopp_encoding(ROCJITSU_CODE_ARCH_INVALID, 0, 0);
static_assert(invalid.failed());

for (int value = 0; value < ROCJITSU_CODE_ARCH_NUM_ARCHS; ++value) {
const auto arch = static_cast<rj_code_arch_t>(value);
SCOPED_TRACE(value);
if (arch == ROCJITSU_CODE_ARCH_RV32I || arch == ROCJITSU_CODE_ARCH_RV64I)
continue;
EXPECT_TRUE(build_sopp_encoding(arch, sopp_op_nop(arch), 0).succeeded());
}

for (auto arch : {ROCJITSU_CODE_ARCH_INVALID, ROCJITSU_CODE_ARCH_RV32I, ROCJITSU_CODE_ARCH_RV64I,
static_cast<rj_code_arch_t>(15)}) {
SCOPED_TRACE(static_cast<int>(arch));
EXPECT_TRUE(build_sopp_encoding(arch, 0, 0).failed());
}
}

TEST(InstructionBuilder, SoppEncodingEmitsOnlyRequestedFailureDiagnostic) {
unsigned messages = 0;
std::string message;
auto callback = [&](std::string_view text) {
++messages;
message = text;
};
const util::DiagnosticEmitter error(callback);
EXPECT_TRUE(build_sopp_encoding(ROCJITSU_CODE_ARCH_CDNA5, 0, 0, &error).succeeded());
EXPECT_EQ(messages, 0u);
EXPECT_TRUE(build_sopp_encoding(ROCJITSU_CODE_ARCH_INVALID, 0, 0, &error).failed());
EXPECT_EQ(messages, 1u);
EXPECT_EQ(message, "unsupported target architecture for SOPP encoding");
}

TEST(InstructionBuilder, DelayAluReturnsEncodingOrFailure) {
constexpr auto rdna4 = build_s_delay_alu(kDelayAluSaluDep1, ROCJITSU_CODE_ARCH_RDNA4);
static_assert(rdna4.succeeded());
Expand Down Expand Up @@ -205,7 +239,10 @@ TEST(GeneratedInstructionBuilder, Gfx1250ScalarPathsUseGeneratedLayouts) {
constexpr uint16_t kSsrc1 = 9;

constexpr auto sopp = cdna5::build_sopp(cdna5::kSBranchSopp, {.simm16 = kSimm16});
EXPECT_EQ(build_sopp_encoding(ROCJITSU_CODE_ARCH_CDNA5, cdna5::kSBranchSopp, kSimm16), sopp[0]);
constexpr auto encoded_sopp =
build_sopp_encoding(ROCJITSU_CODE_ARCH_CDNA5, cdna5::kSBranchSopp, kSimm16);
static_assert(encoded_sopp.succeeded());
EXPECT_EQ(encoded_sopp.value(), sopp[0]);

constexpr auto sop1 = cdna5::build_sop1(cdna5::kSMovB32Sop1, {.ssrc0 = kSsrc0, .sdst = kSdst});
EXPECT_EQ(build_sop1_encoding(ROCJITSU_CODE_ARCH_CDNA5, cdna5::kSMovB32Sop1, kSdst, kSsrc0),
Expand Down
Loading