Skip to content

[rocjitsu] Classify full Wave32 runs with vector stride checks #14250

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kuhar merged 2 commits into from
Oct 26, 2026
Merged

[rocjitsu] Classify full Wave32 runs with vector stride checks #14250

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7c49120
[rocjitsu] Centralize host target capability detection
kuhar Oct 25, 2026
0cc9970
[rocjitsu] Classify full Wave32 runs with vector stride checks
kuhar Oct 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 13 additions & 12 deletions emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/fp_mode.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include "rocjitsu/isa/arch/amdgpu/shared/output_modifier.h"
#include "rocjitsu/isa/arch/amdgpu/shared/pseudo_scalar.h"
#include "util/data_types.h"
#include "util/host_target.h"

#include <algorithm>
#include <bit>
Expand All @@ -27,7 +28,7 @@
#include <limits>
#include <type_traits>

#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__)
#if RJ_HOST_ARCH_X86
#include <xmmintrin.h>
#endif

Expand Down Expand Up @@ -62,7 +63,7 @@ inline int host_round_mode(uint32_t round_mode) {
}
}

#if defined(__aarch64__)
#if RJ_HOST_ARCH_AARCH64
inline uint64_t read_fpcr() {
uint64_t value;
__asm__ volatile("mrs %0, fpcr" : "=r"(value) : : "memory");
Expand All @@ -77,19 +78,19 @@ inline void write_fpcr(uint64_t value) {
class ScopedFenv {
public:
explicit ScopedFenv(uint32_t round_mode) {
#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__)
#if RJ_HOST_ARCH_X86
saved_mxcsr_ = _mm_getcsr();
#elif defined(__aarch64__)
#elif RJ_HOST_ARCH_AARCH64
saved_fpcr_ = read_fpcr();
#endif
saved_ = std::feholdexcept(&environment_) == 0;
if (saved_)
std::fesetround(host_round_mode(round_mode));
#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__)
#if RJ_HOST_ARCH_X86
constexpr uint32_t kDazMask = 1u << 6;
constexpr uint32_t kFtzMask = 1u << 15;
_mm_setcsr(_mm_getcsr() & ~(kDazMask | kFtzMask));
#elif defined(__aarch64__)
#elif RJ_HOST_ARCH_AARCH64
constexpr uint64_t kFizMask = uint64_t{1} << 0;
constexpr uint64_t kFz16Mask = uint64_t{1} << 19;
constexpr uint64_t kFzMask = uint64_t{1} << 24;
Expand All @@ -103,19 +104,19 @@ class ScopedFenv {
~ScopedFenv() {
if (saved_)
std::fesetenv(&environment_);
#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__)
#if RJ_HOST_ARCH_X86
_mm_setcsr(saved_mxcsr_);
#elif defined(__aarch64__)
#elif RJ_HOST_ARCH_AARCH64
write_fpcr(saved_fpcr_);
#endif
}

private:
std::fenv_t environment_{};
bool saved_ = false;
#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__)
#if RJ_HOST_ARCH_X86
uint32_t saved_mxcsr_ = 0;
#elif defined(__aarch64__)
#elif RJ_HOST_ARCH_AARCH64
uint64_t saved_fpcr_ = 0;
#endif
};
Expand Down Expand Up @@ -972,10 +973,10 @@ inline uint16_t finish_arithmetic_f16(double value, uint32_t round_mode, uint32_
inline bool native_arithmetic_matches(uint32_t round_mode, uint32_t denorm_mode) {
if (denorm_mode != 3 || round_mode != 0 || std::fegetround() != FE_TONEAREST)
return false;
#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__)
#if RJ_HOST_ARCH_X86
// MXCSR rounding can differ from the x87 rounding reported by fegetround().
return (_mm_getcsr() & (_MM_ROUND_MASK | (1u << 6) | (1u << 15))) == 0;
#elif defined(__aarch64__)
#elif RJ_HOST_ARCH_AARCH64
return (detail::read_fpcr() & ((uint64_t{1} << 0) | (uint64_t{1} << 19) | (uint64_t{1} << 24))) ==
0;
#else
Expand Down
6 changes: 3 additions & 3 deletions emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/mixed_fma_simd.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ namespace rocjitsu::amdgpu {

// The compatible ABI avoids broken Clang/libstdc++ AVX-512 64-bit masks
// while retaining packed arithmetic instead of a per-lane fallback.
using MixedFmaAbi = std::conditional_t<UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS,
util::stdx::simd_abi::compatible<double>,
util::stdx::simd_abi::native<double>>;
using MixedFmaAbi =
std::conditional_t<RJ_SIMD_BROKEN_NATIVE_64BIT_MASKS, util::stdx::simd_abi::compatible<double>,
util::stdx::simd_abi::native<double>>;
using MixedFmaDouble = util::stdx::simd<double, MixedFmaAbi>;
using MixedFmaBits = util::stdx::simd<uint64_t, MixedFmaAbi>;
inline constexpr std::size_t mixed_fma_width64 = MixedFmaDouble::size();
Expand Down
9 changes: 5 additions & 4 deletions emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/mma_exec.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
#include "rocjitsu/vm/amdgpu/register_access.h"
#include "util/data_types.h"
#include "util/except.h"
#include "util/host_target.h"
#include "util/meta_programming.h"
#include "util/simd.h"

Expand Down Expand Up @@ -3010,7 +3011,7 @@ enum class SwmmacK128Result { F32, F16 };
/// AVX-512 fast path for gfx1250 16x16x128 FP8/BF8 SWMMAC. The sparse
/// selectors occupy two VGPRs per lane. Snapshot both words and every matrix
/// operand before writing D so D may alias A, B, C, or the index tuple.
#if defined(__AVX512F__) && __has_include(<experimental/simd>)
#if RJ_HOST_HAS_AVX512_STDX_SIMD
RJ_NOINLINE inline void exec_swmmac_16x16x128_8bit_fast_body(
auto &cu, uint32_t dst, uint32_t s0, uint32_t s1, uint32_t acc_base, uint32_t index_base,
SwmmacK128Input a_format, SwmmacK128Input b_format, SwmmacK128Result result_format,
Expand Down Expand Up @@ -3152,7 +3153,7 @@ template <typename ExtractA, typename ExtractB>
uint32_t s1, uint32_t acc_base, uint32_t index_base, uint32_t index_entries, uint32_t index_key,
ExtractA, ExtractB, SwmmacK128Result result_format, uint32_t const_acc = ACC_FROM_VGPR,
uint32_t wave_size = WMMA_WAVE32, bool fp16_ovfl = false) {
#if defined(__AVX512F__) && __has_include(<experimental/simd>)
#if RJ_HOST_HAS_AVX512_STDX_SIMD
using A = std::remove_cvref_t<ExtractA>;
using B = std::remove_cvref_t<ExtractB>;
constexpr bool a_fp8 = std::is_same_v<A, ExtractFp8> || std::is_same_v<A, ExtractFp8Ocp>;
Expand Down Expand Up @@ -5331,7 +5332,7 @@ inline float smfmac_decode_snapshot(const uint32_t (&words)[Words], uint32_t ele
/// publishing D: tied accumulators and overlapping A/B/index registers then
/// have the same read-before-write behavior as the scalar helpers. Selected K
/// positions are resolved once per row; the FMA loop retains q,s order.
#if defined(__AVX512F__) && __has_include(<experimental/simd>)
#if RJ_HOST_HAS_AVX512_STDX_SIMD
template <SmfmacLayout Layout, uint32_t M, uint32_t N, uint32_t K, typename ExtractA,
typename ExtractB>
RJ_NOINLINE inline void smfmac_avx512_fast_body(auto &cu, uint32_t dst, uint32_t s0, uint32_t s1,
Expand Down Expand Up @@ -5427,7 +5428,7 @@ template <SmfmacLayout Layout, uint32_t M, uint32_t N, uint32_t K, typename Extr
[[gnu::always_inline]] inline bool smfmac_try_avx512(auto &cu, uint32_t dst, uint32_t s0,
uint32_t s1, uint32_t idx_base, ExtractA,
ExtractB) {
#if defined(__AVX512F__) && __has_include(<experimental/simd>)
#if RJ_HOST_HAS_AVX512_STDX_SIMD
if (util::force_scalar() || cu.wf_size() != 64 || util::native<float>::size() != 16)
return false;

Expand Down
4 changes: 2 additions & 2 deletions emulation/rocjitsu/lib/rocjitsu/src/rocjitsu/isa/arch/amdgpu/shared/simd_coverage.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ template <unsigned Arity, bool E32, bool HalfDst, unsigned HalfInputs, bool Scop
/// requiring subnormal rounding, overflow or exceptional-value handling use
/// the scalar architectural primitive.
template <typename Float>
requires(!(UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS && sizeof(Float) == 8))
requires(!(RJ_SIMD_BROKEN_NATIVE_64BIT_MASKS && sizeof(Float) == 8))
inline auto div_scale_simd(util::native<Float> value, util::native<Float> denominator,
util::native<Float> numerator, uint32_t rounding, uint32_t denorm) {
using F = DivisionFormat<Float>;
Expand Down Expand Up @@ -195,7 +195,7 @@ inline auto div_scale_simd(util::native<Float> value, util::native<Float> denomi
return std::pair{std::bit_cast<util::native<Float>>(result), post_bits};
}

#if UTIL_SIMD_BROKEN_NATIVE_64BIT_MASKS
#if RJ_SIMD_BROKEN_NATIVE_64BIT_MASKS
template <typename Float>
requires(sizeof(Float) == 8)
inline auto div_scale_simd(util::native<Float> value, util::native<Float> denominator,
Expand Down
Loading
Loading