From c2ccdc24c1042e4b8c3b3db5bf42783eb63b4bef Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk <3836908+ozturkosu@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:54:15 -0400 Subject: [PATCH 01/24] feat(ck-tile): enable grouped rowcolquant/tensorquant GEMM bridges on gfx1250 (MI400) Re-applies the gfx1250 enablement of PR #11043 on top of current develop. The original branch predates #10439, which merged the bridges themselves into develop, so only the gfx1250 delta is carried over here. - codegen_common.py: add ROWCOL_TENSOR_QUANT_DEFAULT_TILE_GFX12 and the rowcol_tensor_quant_default_tile(gfx_arch) selector. The stock default tile is the gfx9 MFMA 32x32x16 fragment, which does not exist on gfx12 WMMA hardware: the kernel compiles but produces all-zero output. gfx12 needs the FlatMM 8-bit tile 16x64x256 / warp 1x4x1 / warp_tile 16x16x128. - grouped_gemm_{rowcolquant,tensorquant}_utils.py: _default_config() selects the tile through that helper. gfx942/gfx950 kernel names are unchanged. - grouped_gemm_{rowcolquant,tensorquant}_ctypes_lib.cpp: add gfx1250 to kSupportedArchs, which otherwise rejects the .so in dispatcher_initialize with "compile-time GFX_ARCH 'gfx1250' is not a supported architecture". - test_{rowcolquant,tensorquant}_gpu_correctness.py: add gfx1250 to _SUPPORTED_ARCHES, and default --gfx to the detected device instead of a hardcoded gfx950 (which built a gfx950 image and aborted on any other GPU). Validated on gfx1250 / MI400 (ROCm 10.1): both harnesses PASS fp8 + bf8. --- .../grouped_gemm_rowcolquant_ctypes_lib.cpp | 2 +- .../grouped_gemm_tensorquant_ctypes_lib.cpp | 2 +- .../dispatcher/codegen/codegen_common.py | 23 +++++++++++++++++++ .../python/grouped_gemm_rowcolquant_utils.py | 4 ++-- .../python/grouped_gemm_tensorquant_utils.py | 4 ++-- .../tests/test_rowcolquant_gpu_correctness.py | 19 +++++++++++---- .../tests/test_tensorquant_gpu_correctness.py | 19 +++++++++++---- 7 files changed, 57 insertions(+), 16 deletions(-) diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp index 0f0b4a6b6819..d021bbece8df 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp +++ b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp @@ -74,7 +74,7 @@ static std::atomic g_ref_count{0}; // Architectures this bridge is known to work on. These fp8/bf8 CompV3 kernels need // native FP8, so gfx90a is deliberately absent -- it compiles but produces NaN. // Enabling a new target is a one-line addition here plus a CMake arch entry. -static constexpr const char* kSupportedArchs[] = {"gfx942", "gfx950"}; +static constexpr const char* kSupportedArchs[] = {"gfx942", "gfx950", "gfx1250"}; // True if `arch` starts with any entry of kSupportedArchs. Prefix-matching, because // hipDeviceProp_t::gcnArchName carries feature suffixes (e.g. "gfx942:sramecc+:xnack-"). diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp index 321de7ef2e42..990a62896cbf 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp +++ b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp @@ -77,7 +77,7 @@ static std::atomic g_ref_count{0}; // Architectures this bridge is known to work on. These fp8/bf8 CompV3 kernels need // native FP8, so gfx90a is deliberately absent -- it compiles but produces NaN. // Enabling a new target is a one-line addition here plus a CMake arch entry. -static constexpr const char* kSupportedArchs[] = {"gfx942", "gfx950"}; +static constexpr const char* kSupportedArchs[] = {"gfx942", "gfx950", "gfx1250"}; // True if `arch` starts with any entry of kSupportedArchs. Prefix-matching, because // hipDeviceProp_t::gcnArchName carries feature suffixes (e.g. "gfx942:sramecc+:xnack-"). diff --git a/projects/composablekernel/dispatcher/codegen/codegen_common.py b/projects/composablekernel/dispatcher/codegen/codegen_common.py index ac599a86ffa8..3673e87140c3 100644 --- a/projects/composablekernel/dispatcher/codegen/codegen_common.py +++ b/projects/composablekernel/dispatcher/codegen/codegen_common.py @@ -980,6 +980,29 @@ def make_abquant_kernel_name( "warp_tile_m": 32, "warp_tile_n": 32, "warp_tile_k": 16, } +# gfx12xx (RDNA-style WMMA, e.g. gfx1250/MI400) cannot use the tile above: it is sized +# for the gfx9 MFMA 32x32x16 fragment, which does not exist on WMMA hardware, so the +# kernel compiles but produces all-zero output. The 8-bit WMMA fragment is 16x16x128, +# and the FlatMM 8-bit tile below is the shape validated against it. +ROWCOL_TENSOR_QUANT_DEFAULT_TILE_GFX12 = { + "tile_m": 16, "tile_n": 64, "tile_k": 256, + "warp_m": 1, "warp_n": 4, "warp_k": 1, + "warp_tile_m": 16, "warp_tile_n": 16, "warp_tile_k": 128, +} + + +def rowcol_tensor_quant_default_tile(gfx_arch: str = "") -> dict: + """Return the default RowColQuant/TensorQuant tile for `gfx_arch`. + + Kept here, next to the tile dicts themselves, so the rowcolquant and tensorquant + runtime helpers select the arch-specific tile through one shared code path rather + than each carrying its own copy of the gfx12 shape. + """ + if "gfx12" in gfx_arch: + return dict(ROWCOL_TENSOR_QUANT_DEFAULT_TILE_GFX12) + return dict(ROWCOL_TENSOR_QUANT_DEFAULT_TILE) + + # Default traits, shared for the same reason as the tile above. pad_m is enabled # because these kernels are used with M values that are not tile-aligned. ROWCOL_TENSOR_QUANT_DEFAULT_TRAITS = { diff --git a/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py b/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py index 7bb7d1338770..fd0825748475 100644 --- a/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py +++ b/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py @@ -49,8 +49,8 @@ # their name builders, and keeps the shared tile/trait defaults in one place so this # module's default_*_config() cannot drift from the codegen's _default_config(). from codegen_common import ( # noqa: E402 - ROWCOL_TENSOR_QUANT_DEFAULT_TILE, ROWCOL_TENSOR_QUANT_DEFAULT_TRAITS, + rowcol_tensor_quant_default_tile, make_rowcolquant_kernel_name, ) @@ -676,7 +676,7 @@ def _default_config(dtype: str, gfx_arch: str) -> RowColQuantKernelConfig: pipeline=traits["pipeline"], epilogue=traits["epilogue"], scheduler=traits["scheduler"], - **ROWCOL_TENSOR_QUANT_DEFAULT_TILE, + **rowcol_tensor_quant_default_tile(gfx_arch), pad_m=traits["pad_m"], pad_n=traits["pad_n"], pad_k=traits["pad_k"], diff --git a/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py b/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py index e240b0f9604a..ee0a7eb096da 100644 --- a/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py +++ b/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py @@ -49,8 +49,8 @@ # their name builders, and keeps the shared tile/trait defaults in one place so this # module's default_*_config() cannot drift from the codegen's _default_config(). from codegen_common import ( # noqa: E402 - ROWCOL_TENSOR_QUANT_DEFAULT_TILE, ROWCOL_TENSOR_QUANT_DEFAULT_TRAITS, + rowcol_tensor_quant_default_tile, make_tensorquant_kernel_name, ) @@ -664,7 +664,7 @@ def _default_config(dtype: str, gfx_arch: str) -> TensorQuantKernelConfig: pipeline=traits["pipeline"], epilogue=traits["epilogue"], scheduler=traits["scheduler"], - **ROWCOL_TENSOR_QUANT_DEFAULT_TILE, + **rowcol_tensor_quant_default_tile(gfx_arch), pad_m=traits["pad_m"], pad_n=traits["pad_n"], pad_k=traits["pad_k"], diff --git a/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py index 07a6c49df8f9..34bdf3f729eb 100644 --- a/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py @@ -6,7 +6,7 @@ """ GPU correctness tests for RowColQuant GEMM dispatcher. -Requires a gfx942 or gfx950 GPU and hipcc in PATH. Skipped automatically when neither +Requires a gfx942, gfx950 or gfx1250 GPU and hipcc in PATH. Skipped automatically when neither is available (pytest.skip) so CI without a GPU still passes. Tests: @@ -69,8 +69,8 @@ def _detect_gfx_arch() -> str: _GFX_ARCH = _detect_gfx_arch() # RowColQuant fp8/bf8 kernels use CK CompV3 pipelines that require native fp8 hardware. # gfx90a (MI200 series) lacks native fp8 support and produces incorrect results. -# Only gfx942 (MI300X) and gfx950 (MI350X) are validated. -_SUPPORTED_ARCHES = ("gfx942", "gfx950") +# Only gfx942 (MI300X), gfx950 (MI350X) and gfx1250 (MI400) are validated. +_SUPPORTED_ARCHES = ("gfx942", "gfx950", "gfx1250") requires_gpu = pytest.mark.skipif( not (_has_hipcc() and _GFX_ARCH in _SUPPORTED_ARCHES), @@ -298,7 +298,10 @@ def test_rowcolquant_timing_positive(tmp_path): def main(): parser = argparse.ArgumentParser( description="RowColQuant GPU correctness tests") - parser.add_argument("--gfx", default="gfx950") + # Default to the arch of the GPU we are actually on. A hardcoded default builds a + # gfx950 image on any other device, which then aborts with "device kernel image is + # invalid" (or is rejected by the .so's compile-vs-runtime arch guard). + parser.add_argument("--gfx", default=None) parser.add_argument("-v", "--verbose", action="store_true") parser.add_argument("--output-dir", type=Path, default=None) args = parser.parse_args() @@ -308,6 +311,12 @@ def main(): format="%(levelname)s: %(message)s", ) + gfx = args.gfx or _GFX_ARCH + if not gfx: + log.error("no GPU detected and --gfx not given") + return 1 + log.info("Target arch: %s", gfx) + out_dir = args.output_dir or Path(tempfile.mkdtemp(prefix="rowcolquant_gpu_test_")) log.info("Kernel output dir: %s", out_dir) @@ -315,7 +324,7 @@ def main(): for name, fn in TESTS: log.info("--- Running %s ---", name) try: - status, detail = fn(out_dir, args.gfx) + status, detail = fn(out_dir, gfx) except Exception as exc: status, detail = "FAIL", f"{name}: exception: {exc}" results.append((name, status, detail)) diff --git a/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py index 0fbaa9d8221d..12c67d54b901 100644 --- a/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py @@ -6,7 +6,7 @@ """ GPU correctness tests for TensorQuant GEMM dispatcher. -Requires a gfx942 or gfx950 GPU and hipcc in PATH. Skipped automatically when neither +Requires a gfx942, gfx950 or gfx1250 GPU and hipcc in PATH. Skipped automatically when neither is available (pytest.skip) so CI without a GPU still passes. Tests: @@ -69,8 +69,8 @@ def _detect_gfx_arch() -> str: _GFX_ARCH = _detect_gfx_arch() # TensorQuant fp8/bf8 kernels use CK CompV3 pipelines that require native fp8 hardware. # gfx90a (MI200 series) lacks native fp8 support and produces incorrect results. -# Only gfx942 (MI300X) and gfx950 (MI350X) are validated. -_SUPPORTED_ARCHES = ("gfx942", "gfx950") +# Only gfx942 (MI300X), gfx950 (MI350X) and gfx1250 (MI400) are validated. +_SUPPORTED_ARCHES = ("gfx942", "gfx950", "gfx1250") requires_gpu = pytest.mark.skipif( not (_has_hipcc() and _GFX_ARCH in _SUPPORTED_ARCHES), @@ -296,7 +296,10 @@ def test_tensorquant_timing_positive(tmp_path): def main(): parser = argparse.ArgumentParser( description="TensorQuant GPU correctness tests") - parser.add_argument("--gfx", default="gfx950") + # Default to the arch of the GPU we are actually on. A hardcoded default builds a + # gfx950 image on any other device, which then aborts with "device kernel image is + # invalid" (or is rejected by the .so's compile-vs-runtime arch guard). + parser.add_argument("--gfx", default=None) parser.add_argument("-v", "--verbose", action="store_true") parser.add_argument("--output-dir", type=Path, default=None) args = parser.parse_args() @@ -306,6 +309,12 @@ def main(): format="%(levelname)s: %(message)s", ) + gfx = args.gfx or _GFX_ARCH + if not gfx: + log.error("no GPU detected and --gfx not given") + return 1 + log.info("Target arch: %s", gfx) + out_dir = args.output_dir or Path(tempfile.mkdtemp(prefix="tensorquant_gpu_test_")) log.info("Kernel output dir: %s", out_dir) @@ -313,7 +322,7 @@ def main(): for name, fn in TESTS: log.info("--- Running %s ---", name) try: - status, detail = fn(out_dir, args.gfx) + status, detail = fn(out_dir, gfx) except Exception as exc: status, detail = "FAIL", f"{name}: exception: {exc}" results.append((name, status, detail)) From 32f66c6e9d8bd672373be3ff6438a0cc715ce30d Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk <3836908+ozturkosu@users.noreply.github.com> Date: Thu, 3 Sep 2026 00:38:35 -0400 Subject: [PATCH 02/24] fix(ck-tile): harden gfx1250 rowcolquant/tensorquant bridges against silent failures Addresses review feedback on #11043 plus two defects found while validating the enablement on gfx1250 hardware. All three share a failure mode: the code was wrong in a way that reported success. 1. Arch gate silently skipped on supported devices. The GPU correctness tests gated on `_GFX_ARCH in _SUPPORTED_ARCHES`, an exact string match. rocm_agent_enumerator may report feature suffixes, so a supported device enumerating as "gfx942:sramecc+:xnack-" caused the tests to SKIP. A skip is not a failure, so CI reported green while running none of the GPU coverage. Both test modules now normalize the arch (strip at the first ':') and compare the whole base token, matching the prefix tolerance the C++ bridges already had in is_supported_arch(). Token comparison rather than startswith, so a hypothetical "gfx9421" is not accepted on the strength of the "gfx942" prefix. Pinned by a new CPU-only test module. 2. RowColQuant silently zeroed row M-1 of C when M % 4 != 0 on gfx1250. MakeAQBlockWindow() builds the AQ view as a broadcast make_naive_tensor_view({M, N}, {1, 0}) and then creates the tile window with no padding transform, unlike the A/B/C windows which pad from kPadM/kPadN/kPadK. The M axis therefore has an unguarded tail. TensorQuant is correct at the same M because it passes scalar scales; the BQ view has the same unguarded tail but is unreachable because pad_n=false already forces N % 64 == 0. The root-cause fix belongs in the shared ck_tile quant kernel header, which is also compiled for gfx942/gfx950 -- untested here. Rather than change tile distribution for architectures that were not measured, the bridge now rejects these shapes with an explicit message. The guard is scoped to gfx12 targets via the compile-time GFX_ARCH and to RowColQuant only, and its comment records the condition for removing it. 3. Undocumented argument constraints returned a bare -2. N % TileN is now pre-checked in both bridges, derived from the kernel's own kPadN/TileN constants rather than hardcoded. The -2 message enumerates the known constraints, including that K % 16 == 0 is required even with pad_k=true (pad_k covers the K-loop tail, not the global-load vector width). Constraints documented on the ABI comment and in both run() docstrings, and the Python RuntimeError is now self-contained instead of deferring to stderr. K % 16 is reported but not hard-enforced: it is checked inside ck_tile's IsSupportedArgument() and has not been shown to be general across dtypes and tiles, so enforcing it could reject shapes the kernel would accept. Verified on gfx1250 / MI400: both GPU correctness harnesses 2/2 at max_rel_err=0.0005; every M % 4 == 0 shape unchanged at 2.2e-4 to 3.9e-4; every M % 4 != 0 shape now a clean rejection instead of a wrong answer; TensorQuant still correct at those same M. Host-side suite 59 -> 95 passing, 0 failures. clang-format-18 clean. Known gap, unchanged: the codegen's _default_config() is still arch-agnostic, so a CMake-built lib on gfx1250 would compile the gfx9 tile. Only the Python runtime path is arch-aware. Tracked as a follow-up. --- .../grouped_gemm_rowcolquant_ctypes_lib.cpp | 84 ++++++++++++- .../grouped_gemm_tensorquant_ctypes_lib.cpp | 37 +++++- .../python/grouped_gemm_rowcolquant_utils.py | 25 +++- .../python/grouped_gemm_tensorquant_utils.py | 22 +++- .../tests/test_gpu_test_arch_gate.py | 113 ++++++++++++++++++ .../tests/test_rowcolquant_gpu_correctness.py | 32 ++++- .../tests/test_tensorquant_gpu_correctness.py | 32 ++++- 7 files changed, 333 insertions(+), 12 deletions(-) create mode 100644 projects/composablekernel/dispatcher/tests/test_gpu_test_arch_gate.py diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp index d021bbece8df..d23ddfb589eb 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp +++ b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp @@ -88,6 +88,11 @@ static bool is_supported_arch(const std::string& arch) return false; } +// True if this .so was compiled for a gfx12-class target (WMMA, wave32). +// Used to scope the M%4 rejection below to the architecture where the defect is +// actually observed, so gfx942/gfx950 callers are not restricted by it. +static bool is_gfx12_arch(const std::string& arch) { return arch.rfind("gfx12", 0) == 0; } + extern "C" { /** @@ -166,7 +171,21 @@ int dispatcher_init() { return dispatcher_initialize(); } * k_batch - split-K factor (1 = no split) * time_ms - output: kernel execution time in ms (may be NULL) * - * Returns 0 on success, negative on error. + * Shape constraints (validated on gfx1250 with the default config; violations are + * rejected with an explanatory message and no output is written): + * K % 16 == 0 - required even when the kernel was generated with pad_k=true. + * pad_k covers the K-loop tail, not the global-load vector width. + * N % TileN == 0 - required whenever the kernel was generated with pad_n=false. + * TileN is 64 in the default config. + * M % 4 == 0 - gfx12 targets only. The RowColQuant per-row A-scale tile window + * has an unpadded M tail that zeroes row M-1 of C. See the guard + * in the body for the full explanation; gfx942/gfx950 are not + * subject to this restriction. + * + * Returns 0 on success, negative on error: + * -1 argument rejected by this bridge (including the constraints above) + * -2 argument rejected by the kernel's own IsSupportedArgument() + * -3 kernel launch threw */ int dispatcher_run_gemm(const void* A, const void* B, @@ -224,6 +243,57 @@ int dispatcher_run_gemm(const void* A, return -1; } + // --------------------------------------------------------------------------- + // Documented shape constraints. + // + // These were established empirically on gfx1250 with the default config and are + // enforced here so a violation produces an explanation instead of either a bare + // "unsupported args" code or, worse, a wrong answer. + // --------------------------------------------------------------------------- + + // N must be a whole number of N-tiles unless the kernel pads N. This is derived + // from the kernel's own constants rather than hardcoded, so it stays correct if + // the tile or the padding trait changes. + if(!SelectedKernel::kPadN && (N % SelectedKernel::TileN) != 0) + { + std::cerr << "dispatcher_run_gemm: N must be a multiple of " << SelectedKernel::TileN + << " for this kernel (it was generated with pad_n=false, so N has to be a " + << "whole number of N-tiles); got N=" << N << "\n"; + return -1; + } + + // gfx12 only: the RowColQuant per-row A-scale path drops the final row of C when + // M is not a multiple of 4, writing zeros into row M-1 while leaving every other + // row correct to ~3e-4. The kernel returns success, so without this check the + // caller gets a silent wrong answer -- at M=1 the entire output is zero. + // + // Cause: MakeAQBlockWindow() builds the AQ view with + // make_naive_tensor_view(aq_ptr, {M, N}, {1, 0}, ...) + // and then, as its own comment says, creates the tile window with "no padding for + // AQ" -- unlike the A/B/C windows, which get right-pad transforms driven by + // kPadM/kPadN/kPadK. The M axis of that broadcast view therefore has an unguarded + // tail. The BQ view is the mirror image ({M, N} strided {0, 1}) and has the same + // unguarded tail on N; it is simply never exercised, because pad_n=false already + // forces N to a multiple of TileN (64) and hence of 4. + // + // The fix belongs in ck_tile's shared quant kernel header, not in this bridge: + // that header is common to gfx942/gfx950, where the behaviour has not been + // measured. Rejecting here is deliberately the narrow, reversible option -- it + // converts a silent wrong answer into a clean refusal without touching code paths + // that other architectures depend on. Remove this guard once the AQ tail is + // padded upstream. + if(is_gfx12_arch(std::string(GFX_ARCH)) && (M % 4) != 0) + { + std::cerr << "dispatcher_run_gemm: M must be a multiple of 4 on " << GFX_ARCH + << " for RowColQuant; got M=" << M << ". The per-row A-scale (AQ) tile " + << "window is built without a padding transform, so row M-1 of C is " + << "written as zero while the remaining rows are correct. Refusing rather " + << "than returning a silently wrong result. Pad M up to a multiple of 4 " + << "and slice the result, or use the TensorQuant bridge, which applies " + << "scalar scales and is unaffected.\n"; + return -1; + } + // Only packed (contiguous) layouts are supported for A, B, C. // stride_AQ and stride_BQ are unused here; the kernel uses broadcast strides (0). // @@ -429,7 +499,17 @@ int dispatcher_run_gemm(const void* A, if(exec_time < 0.0f) { - std::cerr << "dispatcher_run_gemm: kernel reported unsupported args\n"; + // IsSupportedArgument() is a boolean inside ck_tile and gives no reason, so + // enumerate the constraints a caller is realistically violating. Observed on + // gfx1250 with the default config: K must be a multiple of 16 (this holds even + // with pad_k=true -- padding covers the K-loop tail, not the global-load vector + // width), and N a multiple of the N-tile when pad_n is false. + std::cerr << "dispatcher_run_gemm: kernel reported unsupported args for M=" << M + << " N=" << N << " K=" << K << " k_batch=" << k_batch + << ". Known constraints for this kernel: K must be a multiple of 16 " + << "(required even though pad_k=" << (SelectedKernel::kPadK ? "true" : "false") + << "); N must be a multiple of " << SelectedKernel::TileN + << " when pad_n is false. No output was written.\n"; cleanup(); return -2; } diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp index 990a62896cbf..f31d13b3beae 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp +++ b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp @@ -165,7 +165,19 @@ int dispatcher_init() { return dispatcher_initialize(); } * k_batch - split-K factor (1 = no split) * time_ms - output: kernel execution time in ms (may be NULL) * - * Returns 0 on success, negative on error. + * Shape constraints (validated on gfx1250 with the default config; violations are + * rejected with an explanatory message and no output is written): + * K % 16 == 0 - required even when the kernel was generated with pad_k=true. + * pad_k covers the K-loop tail, not the global-load vector width. + * N % TileN == 0 - required whenever the kernel was generated with pad_n=false. + * TileN is 64 in the default config. + * There is no M constraint: TensorQuant applies scalar A/B scales rather than a + * per-row scale vector, and is correct at every M tested on gfx1250. + * + * Returns 0 on success, negative on error: + * -1 argument rejected by this bridge (including the constraints above) + * -2 argument rejected by the kernel's own IsSupportedArgument() + * -3 kernel launch threw */ int dispatcher_run_gemm(const void* A, const void* B, @@ -220,6 +232,17 @@ int dispatcher_run_gemm(const void* A, return -1; } + // N must be a whole number of N-tiles unless the kernel pads N. Derived from the + // kernel's own constants rather than hardcoded, so it stays correct if the tile or + // the padding trait changes. Without this the caller only sees a bare -2. + if(!SelectedKernel::kPadN && (N % SelectedKernel::TileN) != 0) + { + std::cerr << "dispatcher_run_gemm: N must be a multiple of " << SelectedKernel::TileN + << " for this kernel (it was generated with pad_n=false, so N has to be a " + << "whole number of N-tiles); got N=" << N << "\n"; + return -1; + } + // Only packed (contiguous) layouts are supported. // // B layout (rcr = row-major A, column-major B, row-major C): @@ -416,7 +439,17 @@ int dispatcher_run_gemm(const void* A, if(exec_time < 0.0f) { - std::cerr << "dispatcher_run_gemm: kernel reported unsupported args\n"; + // IsSupportedArgument() is a boolean inside ck_tile and gives no reason, so + // enumerate the constraints a caller is realistically violating. Observed on + // gfx1250 with the default config: K must be a multiple of 16 (this holds even + // with pad_k=true -- padding covers the K-loop tail, not the global-load vector + // width), and N a multiple of the N-tile when pad_n is false. + std::cerr << "dispatcher_run_gemm: kernel reported unsupported args for M=" << M + << " N=" << N << " K=" << K << " k_batch=" << k_batch + << ". Known constraints for this kernel: K must be a multiple of 16 " + << "(required even though pad_k=" << (SelectedKernel::kPadK ? "true" : "false") + << "); N must be a multiple of " << SelectedKernel::TileN + << " when pad_n is false. No output was written.\n"; cleanup(); return -2; } diff --git a/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py b/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py index fd0825748475..d1deb5bb27d0 100644 --- a/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py +++ b/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py @@ -347,6 +347,21 @@ def run(self, A, B, AQ, BQ, problem: RowColQuantGemmProblem, c_dtype=None) -> Ro BQ shape: (N,) dtype: float (per-col B scale) c_dtype numpy dtype for the output C buffer. Defaults to np.float16. Returns RowColQuantGemmResult with C shape (M, N). + + Shape constraints enforced by the bridge (validated on gfx1250 with the + default config). Violations raise RuntimeError; no output is written: + + K % 16 == 0 Required even though the default config sets pad_k=True -- + pad_k covers the K-loop tail, not the global-load vector + width. + N % 64 == 0 Required because the default config sets pad_n=False, so N + must be a whole number of N-tiles (tile_n=64). + M % 4 == 0 gfx12 targets only. The per-row A-scale (AQ) tile window is + built without a padding transform, so row M-1 of C comes + back as zero while every other row is correct. The bridge + refuses these shapes rather than returning a silently wrong + result. gfx942/gfx950 are not restricted. TensorQuant uses + scalar scales and has no M constraint. """ import numpy as np @@ -407,9 +422,17 @@ def run(self, A, B, AQ, BQ, problem: RowColQuantGemmProblem, c_dtype=None) -> Ro ) if rc != 0: + # rc alone is not actionable, and the C++ explanation goes to stderr, which + # a caller capturing only the exception never sees. Restate the constraints + # here so the traceback is self-contained. raise RuntimeError( f"dispatcher_run_gemm failed with code {rc} " - f"for kernel {self.kernel_name}" + f"for kernel {self.kernel_name} at M={M} N={N} K={K}. " + f"(-1 = rejected by the bridge, -2 = rejected by the kernel, " + f"-3 = launch threw.) Shape constraints: K % 16 == 0 (required even " + f"with pad_k=True); N % 64 == 0 when pad_n=False; and on gfx12 targets " + f"M % 4 == 0, because the per-row AQ tile window is unpadded in M and " + f"would zero row M-1 of C. See stderr for the exact reason." ) return RowColQuantGemmResult(C=C, time_ms=time_ms, kernel_name=self.kernel_name) diff --git a/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py b/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py index ee0a7eb096da..f734a761db8d 100644 --- a/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py +++ b/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py @@ -337,6 +337,19 @@ def run(self, A, B, AQ, BQ, problem: TensorQuantGemmProblem, c_dtype=None) -> Te BQ shape: (1,) dtype: float (per-tensor B scale) c_dtype numpy dtype for the output C buffer. Defaults to np.float16. Returns TensorQuantGemmResult with C shape (M, N). + + Shape constraints enforced by the bridge (validated on gfx1250 with the + default config). Violations raise RuntimeError; no output is written: + + K % 16 == 0 Required even though the default config sets pad_k=True -- + pad_k covers the K-loop tail, not the global-load vector + width. + N % 64 == 0 Required because the default config sets pad_n=False, so N + must be a whole number of N-tiles (tile_n=64). + + There is no M constraint: TensorQuant applies scalar A/B scales rather than a + per-row scale vector, so it is unaffected by the unpadded per-row AQ tail that + restricts the RowColQuant bridge to M % 4 == 0 on gfx12. """ import numpy as np @@ -395,9 +408,16 @@ def run(self, A, B, AQ, BQ, problem: TensorQuantGemmProblem, c_dtype=None) -> Te ) if rc != 0: + # rc alone is not actionable, and the C++ explanation goes to stderr, which + # a caller capturing only the exception never sees. Restate the constraints + # here so the traceback is self-contained. raise RuntimeError( f"dispatcher_run_gemm failed with code {rc} " - f"for kernel {self.kernel_name}" + f"for kernel {self.kernel_name} at M={M} N={N} K={K}. " + f"(-1 = rejected by the bridge, -2 = rejected by the kernel, " + f"-3 = launch threw.) Shape constraints: K % 16 == 0 (required even " + f"with pad_k=True); N % 64 == 0 when pad_n=False. " + f"See stderr for the exact reason." ) return TensorQuantGemmResult(C=C, time_ms=time_ms, kernel_name=self.kernel_name) diff --git a/projects/composablekernel/dispatcher/tests/test_gpu_test_arch_gate.py b/projects/composablekernel/dispatcher/tests/test_gpu_test_arch_gate.py new file mode 100644 index 000000000000..1497dea1a144 --- /dev/null +++ b/projects/composablekernel/dispatcher/tests/test_gpu_test_arch_gate.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 + +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +"""Unit tests for the arch gate used by the RowColQuant/TensorQuant GPU tests. + +These run on CPU: they exercise the pure string predicate, not a device. + +Why this file exists +-------------------- +The GPU correctness tests decide whether to run or skip by comparing the detected +architecture against a tuple of supported targets. That comparison used to be an +exact ``in`` test, which is wrong: ``rocm_agent_enumerator`` (and +``hipDeviceProp_t::gcnArchName``) may report feature suffixes, so a perfectly +supported device can enumerate as ``"gfx942:sramecc+:xnack-"``. An exact compare +then evaluates False and the test SKIPS. + +A skip is not a failure, so CI stays green while covering nothing -- the defect is +invisible by construction. That failure mode is why this predicate gets its own +regression test rather than being trusted to review. +""" + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import test_rowcolquant_gpu_correctness as rcq +import test_tensorquant_gpu_correctness as tq + +# Both modules must gate identically; parametrizing over them keeps them in step. +MODULES = pytest.mark.parametrize("mod", [rcq, tq], ids=["rowcolquant", "tensorquant"]) + + +@MODULES +@pytest.mark.parametrize("arch", ["gfx942", "gfx950", "gfx1250"]) +def test_bare_supported_arch_is_accepted(mod, arch): + assert mod.arch_is_supported(arch) is True + + +@MODULES +@pytest.mark.parametrize( + "arch", + [ + "gfx942:sramecc+:xnack-", + "gfx942:sramecc-:xnack-", + "gfx950:sramecc+:xnack-", + "gfx1250:xnack-", + ], +) +def test_feature_suffixed_supported_arch_is_accepted(mod, arch): + """The regression this file exists for: suffixes must not cause a skip.""" + assert mod.arch_is_supported(arch) is True, ( + f"{arch} is a supported device but the gate rejected it; " + "the GPU test would silently skip and CI would be vacuously green" + ) + + +@MODULES +@pytest.mark.parametrize( + "arch", + [ + "gfx90a", + "gfx90a:sramecc+:xnack-", # unsupported and suffixed -> still unsupported + "gfx1030", + "gfx000", + "", + ], +) +def test_unsupported_arch_is_rejected(mod, arch): + assert mod.arch_is_supported(arch) is False + + +@MODULES +def test_no_substring_false_positives(mod): + """Prefix tolerance must not degrade into a substring match. + + ``gfx9421`` is a different (hypothetical) target and must not be accepted just + because ``gfx942`` is a prefix of it. Splitting on ``:`` and comparing the whole + base token gives us that for free; a naive ``startswith`` would not. + """ + assert mod.arch_is_supported("gfx9421") is False + assert mod.arch_is_supported("gfx12501") is False + + +@MODULES +@pytest.mark.parametrize( + "raw,expected", + [ + ("gfx942:sramecc+:xnack-", "gfx942"), + ("gfx1250", "gfx1250"), + ("gfx950:xnack-", "gfx950"), + ("", ""), + ], +) +def test_normalize_strips_feature_suffix(mod, raw, expected): + """Normalization also feeds ``--offload-arch``, so it must yield a bare target.""" + assert mod.normalize_gfx_arch(raw) == expected + + +@MODULES +def test_gate_matches_supported_tuple(mod): + """Every entry of the declared tuple must pass its own gate, bare and suffixed.""" + for arch in mod._SUPPORTED_ARCHES: + assert mod.arch_is_supported(arch) is True + assert mod.arch_is_supported(f"{arch}:sramecc+:xnack-") is True + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py index 34bdf3f729eb..65ec88e93f05 100644 --- a/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py @@ -53,14 +53,40 @@ def _has_hipcc() -> bool: return shutil.which("hipcc") is not None +def normalize_gfx_arch(arch: str) -> str: + """Strip feature suffixes from a gfx target string. + + ``rocm_agent_enumerator`` and ``hipDeviceProp_t::gcnArchName`` may report the + target with trailing feature flags, e.g. ``"gfx942:sramecc+:xnack-"``. Every + comparison we do here (and the ``--offload-arch`` we hand to hipcc) wants the + bare target, so normalize once at the boundary. + """ + return arch.split(":", 1)[0] + + +def arch_is_supported(arch: str, supported=None) -> bool: + """True if `arch` names a supported target, ignoring feature suffixes. + + An exact ``arch in _SUPPORTED_ARCHES`` test returns False for + ``"gfx942:sramecc+:xnack-"``, so on a machine whose enumerator reports + suffixes these GPU tests would SKIP on a fully supported device. A skip reads + as a green run, so the regression would be invisible -- the bug is the silent + skip, not the missing coverage. The C++ bridge already prefix-matches in + ``is_supported_arch()``; this keeps the Python gate consistent with it. + """ + if supported is None: + supported = _SUPPORTED_ARCHES + return normalize_gfx_arch(arch) in supported + + def _detect_gfx_arch() -> str: """Return the first usable GPU arch, or empty string if none found.""" try: r = subprocess.run(["rocm_agent_enumerator"], capture_output=True, text=True, timeout=10) for line in r.stdout.splitlines(): arch = line.strip() - if arch.startswith("gfx") and arch != "gfx000": - return arch + if arch.startswith("gfx") and normalize_gfx_arch(arch) != "gfx000": + return normalize_gfx_arch(arch) except Exception: pass return "" @@ -73,7 +99,7 @@ def _detect_gfx_arch() -> str: _SUPPORTED_ARCHES = ("gfx942", "gfx950", "gfx1250") requires_gpu = pytest.mark.skipif( - not (_has_hipcc() and _GFX_ARCH in _SUPPORTED_ARCHES), + not (_has_hipcc() and arch_is_supported(_GFX_ARCH)), reason=( f"GPU test: requires hipcc and native fp8 GPU ({', '.join(_SUPPORTED_ARCHES)}); " f"detected arch='{_GFX_ARCH}'" diff --git a/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py index 12c67d54b901..b4e5a5b67725 100644 --- a/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py @@ -53,14 +53,40 @@ def _has_hipcc() -> bool: return shutil.which("hipcc") is not None +def normalize_gfx_arch(arch: str) -> str: + """Strip feature suffixes from a gfx target string. + + ``rocm_agent_enumerator`` and ``hipDeviceProp_t::gcnArchName`` may report the + target with trailing feature flags, e.g. ``"gfx942:sramecc+:xnack-"``. Every + comparison we do here (and the ``--offload-arch`` we hand to hipcc) wants the + bare target, so normalize once at the boundary. + """ + return arch.split(":", 1)[0] + + +def arch_is_supported(arch: str, supported=None) -> bool: + """True if `arch` names a supported target, ignoring feature suffixes. + + An exact ``arch in _SUPPORTED_ARCHES`` test returns False for + ``"gfx942:sramecc+:xnack-"``, so on a machine whose enumerator reports + suffixes these GPU tests would SKIP on a fully supported device. A skip reads + as a green run, so the regression would be invisible -- the bug is the silent + skip, not the missing coverage. The C++ bridge already prefix-matches in + ``is_supported_arch()``; this keeps the Python gate consistent with it. + """ + if supported is None: + supported = _SUPPORTED_ARCHES + return normalize_gfx_arch(arch) in supported + + def _detect_gfx_arch() -> str: """Return the first usable GPU arch, or empty string if none found.""" try: r = subprocess.run(["rocm_agent_enumerator"], capture_output=True, text=True, timeout=10) for line in r.stdout.splitlines(): arch = line.strip() - if arch.startswith("gfx") and arch != "gfx000": - return arch + if arch.startswith("gfx") and normalize_gfx_arch(arch) != "gfx000": + return normalize_gfx_arch(arch) except Exception: pass return "" @@ -73,7 +99,7 @@ def _detect_gfx_arch() -> str: _SUPPORTED_ARCHES = ("gfx942", "gfx950", "gfx1250") requires_gpu = pytest.mark.skipif( - not (_has_hipcc() and _GFX_ARCH in _SUPPORTED_ARCHES), + not (_has_hipcc() and arch_is_supported(_GFX_ARCH)), reason=( f"GPU test: requires hipcc and native fp8 GPU ({', '.join(_SUPPORTED_ARCHES)}); " f"detected arch='{_GFX_ARCH}'" From 36c88c1a6dd6e26aeda454dab7c3451e4f94eda0 Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk <3836908+ozturkosu@users.noreply.github.com> Date: Wed, 9 Sep 2026 17:55:16 -0400 Subject: [PATCH 03/24] fix(ck-tile): exact gfx1250 tile match and dynamic TileN diagnostics The RowColQuant/TensorQuant default-tile selector matched the whole gfx12 family. gfx1200/gfx1201 are WMMA parts too, but their 8-bit warp fragment is 16x16x16, not gfx1250's 16x16x64/16x16x128, so they would have inherited a tile that compiles cleanly and returns garbage. Nothing upstream stops that: the runtime helpers have no arch allowlist. Match gfx1250 exactly instead. The OCP-FP8 hipcc define stays family-wide on purpose -- every gfx12xx part uses OCP FP8 encoding -- and is now written as an explicit family test with a comment, so the two sites cannot be mistaken for the same rule. Add a shared normalize_gfx_arch() in codegen_common as the single source of truth for stripping feature suffixes ("gfx1250:xnack-"), use it at the arch detection boundary so suffixes never reach --offload-arch/-DGFX_ARCH (the C++ side prefix-matches, so the bare target still matches a suffixed device), and drop the duplicated local copies in the two GPU correctness tests. The C++ derives its N-divisibility bound from SelectedKernel::TileN, but the Python docstring and RuntimeError hardcoded 64. Export dispatcher_get_tile_n() and dispatcher_get_pad_n() from both ctypes libs and use the real values, so the diagnostics stay true if the compiled tile changes. Tests cover suffix normalization, exact-vs-family arch matching for the tile selector, and the intentional gfx12-family OCP-FP8 behaviour. --- .../grouped_gemm_rowcolquant_ctypes_lib.cpp | 17 +++++ .../grouped_gemm_tensorquant_ctypes_lib.cpp | 17 +++++ .../dispatcher/codegen/codegen_common.py | 36 ++++++++-- .../python/grouped_gemm_rowcolquant_utils.py | 63 +++++++++++++++-- .../python/grouped_gemm_tensorquant_utils.py | 63 +++++++++++++++-- .../dispatcher/tests/test_codegen_common.py | 68 +++++++++++++++++++ .../test_grouped_gemm_rowcolquant_utils.py | 56 +++++++++++++++ .../test_grouped_gemm_tensorquant_utils.py | 56 +++++++++++++++ .../tests/test_rowcolquant_gpu_correctness.py | 15 ++-- .../tests/test_tensorquant_gpu_correctness.py | 15 ++-- 10 files changed, 365 insertions(+), 41 deletions(-) diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp index d23ddfb589eb..85c2bde948f6 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp +++ b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp @@ -529,6 +529,23 @@ int dispatcher_run_gemm(const void* A, */ const char* dispatcher_get_kernel_name() { return KERNEL_NAME; } +/** + * Return the N-tile of the force-included kernel. + * + * The shape checks in dispatcher_run_gemm derive their bound from SelectedKernel::TileN + * so they survive a tile change. Exporting it lets the Python runner report the same + * number instead of hardcoding the value of today's default config. + */ +int dispatcher_get_tile_n() { return static_cast(SelectedKernel::TileN); } + +/** + * Return 1 when the force-included kernel was generated with pad_n=true. + * + * When padding is on the N % TileN constraint does not apply, so the Python + * runner needs this to phrase its diagnostics correctly. + */ +int dispatcher_get_pad_n() { return SelectedKernel::kPadN ? 1 : 0; } + // This bridge is one-.so-per-kernel by construction: the build force-includes exactly // one generated header via `hipcc -include `, giving one SelectedKernel. // Scaling to N kernels means N .so files (the pattern bquant/aquant/abquant follow), diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp index f31d13b3beae..cc4748d68bd3 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp +++ b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp @@ -469,6 +469,23 @@ int dispatcher_run_gemm(const void* A, */ const char* dispatcher_get_kernel_name() { return KERNEL_NAME; } +/** + * Return the N-tile of the force-included kernel. + * + * The shape checks in dispatcher_run_gemm derive their bound from SelectedKernel::TileN + * so they survive a tile change. Exporting it lets the Python runner report the same + * number instead of hardcoding the value of today's default config. + */ +int dispatcher_get_tile_n() { return static_cast(SelectedKernel::TileN); } + +/** + * Return 1 when the force-included kernel was generated with pad_n=true. + * + * When padding is on the N % TileN constraint does not apply, so the Python + * runner needs this to phrase its diagnostics correctly. + */ +int dispatcher_get_pad_n() { return SelectedKernel::kPadN ? 1 : 0; } + // This bridge is one-.so-per-kernel by construction: the build force-includes exactly // one generated header via `hipcc -include `, giving one SelectedKernel. // Scaling to N kernels means N .so files (the pattern bquant/aquant/abquant follow), diff --git a/projects/composablekernel/dispatcher/codegen/codegen_common.py b/projects/composablekernel/dispatcher/codegen/codegen_common.py index 3673e87140c3..d2581668547a 100644 --- a/projects/composablekernel/dispatcher/codegen/codegen_common.py +++ b/projects/composablekernel/dispatcher/codegen/codegen_common.py @@ -629,6 +629,26 @@ def make_gemm_rowcolquant_kernel_name( ) +# ============================================================================ +# Arch string normalization +# ============================================================================ + + +def normalize_gfx_arch(arch: str) -> str: + """Strip feature suffixes from a gfx target string. + + ``rocm_agent_enumerator`` and ``hipDeviceProp_t::gcnArchName`` may report the + target with trailing feature flags, e.g. ``"gfx942:sramecc+:xnack-"`` or + ``"gfx1250:xnack-"``. Every arch comparison in the codegen/runtime path (and + the ``--offload-arch`` we hand to hipcc) wants the bare target, so normalize + once at the boundary instead of scattering substring tests that happen to + tolerate the suffix. + + This is the single source of truth for that rule; do not re-implement it. + """ + return arch.split(":", 1)[0] + + # ============================================================================ # Arch-derived warp tile K # ============================================================================ @@ -980,11 +1000,11 @@ def make_abquant_kernel_name( "warp_tile_m": 32, "warp_tile_n": 32, "warp_tile_k": 16, } -# gfx12xx (RDNA-style WMMA, e.g. gfx1250/MI400) cannot use the tile above: it is sized +# gfx1250 (RDNA-style WMMA, MI400) cannot use the tile above: it is sized # for the gfx9 MFMA 32x32x16 fragment, which does not exist on WMMA hardware, so the # kernel compiles but produces all-zero output. The 8-bit WMMA fragment is 16x16x128, # and the FlatMM 8-bit tile below is the shape validated against it. -ROWCOL_TENSOR_QUANT_DEFAULT_TILE_GFX12 = { +ROWCOL_TENSOR_QUANT_DEFAULT_TILE_GFX1250 = { "tile_m": 16, "tile_n": 64, "tile_k": 256, "warp_m": 1, "warp_n": 4, "warp_k": 1, "warp_tile_m": 16, "warp_tile_n": 16, "warp_tile_k": 128, @@ -996,10 +1016,16 @@ def rowcol_tensor_quant_default_tile(gfx_arch: str = "") -> dict: Kept here, next to the tile dicts themselves, so the rowcolquant and tensorquant runtime helpers select the arch-specific tile through one shared code path rather - than each carrying its own copy of the gfx12 shape. + than each carrying its own copy of the gfx1250 shape. + + The gfx1250 test is EXACT, not a ``gfx12`` family test. gfx1200/gfx1201 are + also WMMA parts, but their 8-bit warp fragment is 16x16x16, not the 16x16x64 / + 16x16x128 of gfx1250, so handing them the gfx1250 tile would compile cleanly + and return garbage. Contrast the OCP-FP8 define in the rowcolquant/tensorquant + runtime helpers, which *is* correctly family-wide. """ - if "gfx12" in gfx_arch: - return dict(ROWCOL_TENSOR_QUANT_DEFAULT_TILE_GFX12) + if normalize_gfx_arch(gfx_arch) == "gfx1250": + return dict(ROWCOL_TENSOR_QUANT_DEFAULT_TILE_GFX1250) return dict(ROWCOL_TENSOR_QUANT_DEFAULT_TILE) diff --git a/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py b/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py index d1deb5bb27d0..ec469b25e8c5 100644 --- a/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py +++ b/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py @@ -51,6 +51,7 @@ from codegen_common import ( # noqa: E402 ROWCOL_TENSOR_QUANT_DEFAULT_TRAITS, rowcol_tensor_quant_default_tile, + normalize_gfx_arch, make_rowcolquant_kernel_name, ) @@ -189,6 +190,8 @@ class RowColQuantDispatcherLib: QK_A, QK_B, k_batch, *time_ms) char* dispatcher_get_kernel_name() int dispatcher_get_kernel_count() + int dispatcher_get_tile_n() + int dispatcher_get_pad_n() void dispatcher_cleanup() """ @@ -236,6 +239,17 @@ def _setup(self): lib.dispatcher_get_kernel_count.restype = ctypes.c_int lib.dispatcher_get_kernel_count.argtypes = [] + # TileN / pad_n are compile-time properties of the single force-included + # kernel. They are read back rather than hardcoded so the N-divisibility + # diagnostics below stay correct if the default tile changes. No missing- + # symbol fallback: this .so is always compiled from the ctypes source that + # ships these exports, by this module. + lib.dispatcher_get_tile_n.restype = ctypes.c_int + lib.dispatcher_get_tile_n.argtypes = [] + + lib.dispatcher_get_pad_n.restype = ctypes.c_int + lib.dispatcher_get_pad_n.argtypes = [] + lib.dispatcher_cleanup.restype = None lib.dispatcher_cleanup.argtypes = [] @@ -306,6 +320,14 @@ def get_kernel_name(self) -> str: def get_kernel_count(self) -> int: return self._lib.dispatcher_get_kernel_count() + def get_tile_n(self) -> int: + """N-tile of the compiled kernel (SelectedKernel::TileN).""" + return self._lib.dispatcher_get_tile_n() + + def get_pad_n(self) -> bool: + """True when the compiled kernel was generated with pad_n=true.""" + return bool(self._lib.dispatcher_get_pad_n()) + def cleanup(self): if not self._cleaned_up: self._lib.dispatcher_cleanup() @@ -337,6 +359,16 @@ def __init__(self, so_path: Path): def kernel_name(self) -> str: return self._lib.get_kernel_name() + @property + def tile_n(self) -> int: + """N-tile of the compiled kernel, read from the .so (not hardcoded).""" + return self._lib.get_tile_n() + + @property + def pad_n(self) -> bool: + """True when the compiled kernel pads N (the N % tile_n rule then lifts).""" + return self._lib.get_pad_n() + def run(self, A, B, AQ, BQ, problem: RowColQuantGemmProblem, c_dtype=None) -> RowColQuantGemmResult: """ Run RowColQuant Grouped GEMM. @@ -354,8 +386,10 @@ def run(self, A, B, AQ, BQ, problem: RowColQuantGemmProblem, c_dtype=None) -> Ro K % 16 == 0 Required even though the default config sets pad_k=True -- pad_k covers the K-loop tail, not the global-load vector width. - N % 64 == 0 Required because the default config sets pad_n=False, so N - must be a whole number of N-tiles (tile_n=64). + N % tile_n == 0 Required whenever the kernel was generated with pad_n=False, + so N must be a whole number of N-tiles. tile_n is a property + of the compiled kernel -- read `self.tile_n` rather than + assuming the value of today's default config. M % 4 == 0 gfx12 targets only. The per-row A-scale (AQ) tile window is built without a padding transform, so row M-1 of C comes back as zero while every other row is correct. The bridge @@ -424,13 +458,19 @@ def run(self, A, B, AQ, BQ, problem: RowColQuantGemmProblem, c_dtype=None) -> Ro if rc != 0: # rc alone is not actionable, and the C++ explanation goes to stderr, which # a caller capturing only the exception never sees. Restate the constraints - # here so the traceback is self-contained. + # here so the traceback is self-contained. tile_n/pad_n come from the .so, + # so this message stays true if the compiled tile changes. + n_rule = ( + "N is unconstrained (pad_n=True)" + if self.pad_n + else f"N % {self.tile_n} == 0 when pad_n=False" + ) raise RuntimeError( f"dispatcher_run_gemm failed with code {rc} " f"for kernel {self.kernel_name} at M={M} N={N} K={K}. " f"(-1 = rejected by the bridge, -2 = rejected by the kernel, " f"-3 = launch threw.) Shape constraints: K % 16 == 0 (required even " - f"with pad_k=True); N % 64 == 0 when pad_n=False; and on gfx12 targets " + f"with pad_k=True); {n_rule}; and on gfx12 targets " f"M % 4 == 0, because the per-row AQ tile window is unpadded in M and " f"would zero row M-1 of C. See stderr for the exact reason." ) @@ -453,7 +493,11 @@ def _detect_gpu_arch() -> str: for line in result.stdout.splitlines(): line = line.strip() if line.startswith("gfx") and line != "gfx000": - return line + # Strip feature suffixes ("gfx1250:xnack-") here so they never reach + # --offload-arch / -DGFX_ARCH. The C++ side prefix-matches the runtime + # device name against the compile-time GFX_ARCH, so the bare target + # still matches a device that reports suffixes. + return normalize_gfx_arch(line) except Exception as e: log.warning("rocm_agent_enumerator failed (%s); defaulting to %s", e, _DEFAULT_GFX_ARCH) return _DEFAULT_GFX_ARCH @@ -524,9 +568,14 @@ def _compile_rowcolquant_kernel( obj_path = so_path.with_suffix(".o") arch_defines = [] - if "gfx12" in gfx_arch or "gfx950" in gfx_arch: + # Family test on purpose: every gfx12xx part (gfx1200/gfx1201/gfx1250) uses OCP + # FP8 encoding, so an exact-gfx1250 test here would be WRONG. This is the + # opposite of the tile selector in codegen_common.rowcol_tensor_quant_default_tile(), + # which must be exact because gfx1200/gfx1201 have a different 8-bit warp fragment. + _arch_base = normalize_gfx_arch(gfx_arch) + if _arch_base.startswith("gfx12") or _arch_base == "gfx950": arch_defines += ["-DCK_USE_OCP_FP8", "-DCK_TILE_USE_OCP_FP8"] - if "gfx950" in gfx_arch: + if _arch_base == "gfx950": arch_defines += ["-DCK_USE_NATIVE_MX_SUPPORT", "-DCK_GFX950_SUPPORT"] compile_cmd = [hipcc, "-c", "-fPIC", "-O3", "-std=c++17", diff --git a/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py b/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py index f734a761db8d..a9f42aecc833 100644 --- a/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py +++ b/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py @@ -51,6 +51,7 @@ from codegen_common import ( # noqa: E402 ROWCOL_TENSOR_QUANT_DEFAULT_TRAITS, rowcol_tensor_quant_default_tile, + normalize_gfx_arch, make_tensorquant_kernel_name, ) @@ -179,6 +180,8 @@ class TensorQuantDispatcherLib: QK_A, QK_B, k_batch, *time_ms) char* dispatcher_get_kernel_name() int dispatcher_get_kernel_count() + int dispatcher_get_tile_n() + int dispatcher_get_pad_n() void dispatcher_cleanup() """ @@ -226,6 +229,17 @@ def _setup(self): lib.dispatcher_get_kernel_count.restype = ctypes.c_int lib.dispatcher_get_kernel_count.argtypes = [] + # TileN / pad_n are compile-time properties of the single force-included + # kernel. They are read back rather than hardcoded so the N-divisibility + # diagnostics below stay correct if the default tile changes. No missing- + # symbol fallback: this .so is always compiled from the ctypes source that + # ships these exports, by this module. + lib.dispatcher_get_tile_n.restype = ctypes.c_int + lib.dispatcher_get_tile_n.argtypes = [] + + lib.dispatcher_get_pad_n.restype = ctypes.c_int + lib.dispatcher_get_pad_n.argtypes = [] + lib.dispatcher_cleanup.restype = None lib.dispatcher_cleanup.argtypes = [] @@ -296,6 +310,14 @@ def get_kernel_name(self) -> str: def get_kernel_count(self) -> int: return self._lib.dispatcher_get_kernel_count() + def get_tile_n(self) -> int: + """N-tile of the compiled kernel (SelectedKernel::TileN).""" + return self._lib.dispatcher_get_tile_n() + + def get_pad_n(self) -> bool: + """True when the compiled kernel was generated with pad_n=true.""" + return bool(self._lib.dispatcher_get_pad_n()) + def cleanup(self): if not self._cleaned_up: self._lib.dispatcher_cleanup() @@ -327,6 +349,16 @@ def __init__(self, so_path: Path): def kernel_name(self) -> str: return self._lib.get_kernel_name() + @property + def tile_n(self) -> int: + """N-tile of the compiled kernel, read from the .so (not hardcoded).""" + return self._lib.get_tile_n() + + @property + def pad_n(self) -> bool: + """True when the compiled kernel pads N (the N % tile_n rule then lifts).""" + return self._lib.get_pad_n() + def run(self, A, B, AQ, BQ, problem: TensorQuantGemmProblem, c_dtype=None) -> TensorQuantGemmResult: """ Run TensorQuant Grouped GEMM. @@ -344,8 +376,10 @@ def run(self, A, B, AQ, BQ, problem: TensorQuantGemmProblem, c_dtype=None) -> Te K % 16 == 0 Required even though the default config sets pad_k=True -- pad_k covers the K-loop tail, not the global-load vector width. - N % 64 == 0 Required because the default config sets pad_n=False, so N - must be a whole number of N-tiles (tile_n=64). + N % tile_n == 0 Required whenever the kernel was generated with pad_n=False, + so N must be a whole number of N-tiles. tile_n is a property + of the compiled kernel -- read `self.tile_n` rather than + assuming the value of today's default config. There is no M constraint: TensorQuant applies scalar A/B scales rather than a per-row scale vector, so it is unaffected by the unpadded per-row AQ tail that @@ -410,13 +444,19 @@ def run(self, A, B, AQ, BQ, problem: TensorQuantGemmProblem, c_dtype=None) -> Te if rc != 0: # rc alone is not actionable, and the C++ explanation goes to stderr, which # a caller capturing only the exception never sees. Restate the constraints - # here so the traceback is self-contained. + # here so the traceback is self-contained. tile_n/pad_n come from the .so, + # so this message stays true if the compiled tile changes. + n_rule = ( + "N is unconstrained (pad_n=True)" + if self.pad_n + else f"N % {self.tile_n} == 0 when pad_n=False" + ) raise RuntimeError( f"dispatcher_run_gemm failed with code {rc} " f"for kernel {self.kernel_name} at M={M} N={N} K={K}. " f"(-1 = rejected by the bridge, -2 = rejected by the kernel, " f"-3 = launch threw.) Shape constraints: K % 16 == 0 (required even " - f"with pad_k=True); N % 64 == 0 when pad_n=False. " + f"with pad_k=True); {n_rule}. " f"See stderr for the exact reason." ) @@ -438,7 +478,11 @@ def _detect_gpu_arch() -> str: for line in result.stdout.splitlines(): line = line.strip() if line.startswith("gfx") and line != "gfx000": - return line + # Strip feature suffixes ("gfx1250:xnack-") here so they never reach + # --offload-arch / -DGFX_ARCH. The C++ side prefix-matches the runtime + # device name against the compile-time GFX_ARCH, so the bare target + # still matches a device that reports suffixes. + return normalize_gfx_arch(line) except Exception as e: log.warning("rocm_agent_enumerator failed (%s); defaulting to %s", e, _DEFAULT_GFX_ARCH) return _DEFAULT_GFX_ARCH @@ -509,9 +553,14 @@ def _compile_tensorquant_kernel( obj_path = so_path.with_suffix(".o") arch_defines = [] - if "gfx12" in gfx_arch or "gfx950" in gfx_arch: + # Family test on purpose: every gfx12xx part (gfx1200/gfx1201/gfx1250) uses OCP + # FP8 encoding, so an exact-gfx1250 test here would be WRONG. This is the + # opposite of the tile selector in codegen_common.rowcol_tensor_quant_default_tile(), + # which must be exact because gfx1200/gfx1201 have a different 8-bit warp fragment. + _arch_base = normalize_gfx_arch(gfx_arch) + if _arch_base.startswith("gfx12") or _arch_base == "gfx950": arch_defines += ["-DCK_USE_OCP_FP8", "-DCK_TILE_USE_OCP_FP8"] - if "gfx950" in gfx_arch: + if _arch_base == "gfx950": arch_defines += ["-DCK_USE_NATIVE_MX_SUPPORT", "-DCK_GFX950_SUPPORT"] compile_cmd = [hipcc, "-c", "-fPIC", "-O3", "-std=c++17", diff --git a/projects/composablekernel/dispatcher/tests/test_codegen_common.py b/projects/composablekernel/dispatcher/tests/test_codegen_common.py index 6e14b201943e..9389174d5abe 100644 --- a/projects/composablekernel/dispatcher/tests/test_codegen_common.py +++ b/projects/composablekernel/dispatcher/tests/test_codegen_common.py @@ -38,6 +38,10 @@ gemm_aquant_effective_epilogue, make_bquant_kernel_name, make_gemm_rowcolquant_kernel_name, + normalize_gfx_arch, + rowcol_tensor_quant_default_tile, + ROWCOL_TENSOR_QUANT_DEFAULT_TILE, + ROWCOL_TENSOR_QUANT_DEFAULT_TILE_GFX1250, make_aquant_kernel_name, # Shared quant spec-sweep plumbing (see TestQuantSpecSweepHelpers). fp8_warp_tile_k_for_arch, @@ -633,5 +637,69 @@ def test_unknown_arch_takes_the_safe_non_gfx950_branch(self): self.assertEqual(fp8_warp_tile_k_for_arch("gfx90a"), 32) +class TestNormalizeGfxArch(unittest.TestCase): + """Feature suffixes must be stripped exactly once, here.""" + + def test_bare_target_is_unchanged(self): + self.assertEqual(normalize_gfx_arch("gfx1250"), "gfx1250") + self.assertEqual(normalize_gfx_arch("gfx942"), "gfx942") + + def test_single_suffix_is_stripped(self): + self.assertEqual(normalize_gfx_arch("gfx1250:xnack-"), "gfx1250") + + def test_multiple_suffixes_are_stripped(self): + self.assertEqual(normalize_gfx_arch("gfx942:sramecc+:xnack-"), "gfx942") + + def test_unknown_or_malformed_input_is_passed_through(self): + # Not a validator: an unrecognised or empty string must come back + # unchanged (minus any suffix) so the caller's own arch test decides. + self.assertEqual(normalize_gfx_arch(""), "") + self.assertEqual(normalize_gfx_arch("notagfx:weird"), "notagfx") + self.assertEqual(normalize_gfx_arch(":xnack-"), "") + + +class TestRowColTensorQuantDefaultTile(unittest.TestCase): + """gfx1250 tile selection is EXACT, not a gfx12 family match. + + gfx1200/gfx1201 are WMMA parts too, but their 8-bit warp fragment is + 16x16x16, not gfx1250's 16x16x64/16x16x128. Handing them the gfx1250 tile + compiles cleanly and returns garbage, so a family match here is a + silent-wrong-answer bug. + """ + + def test_gfx1250_takes_the_gfx1250_tile(self): + self.assertEqual( + rowcol_tensor_quant_default_tile("gfx1250"), + dict(ROWCOL_TENSOR_QUANT_DEFAULT_TILE_GFX1250), + ) + + def test_gfx1250_with_feature_suffix_takes_the_gfx1250_tile(self): + self.assertEqual( + rowcol_tensor_quant_default_tile("gfx1250:xnack-"), + dict(ROWCOL_TENSOR_QUANT_DEFAULT_TILE_GFX1250), + ) + + def test_other_gfx12_parts_do_not_inherit_the_gfx1250_tile(self): + for arch in ("gfx1200", "gfx1201", "gfx1201:xnack-"): + with self.subTest(arch=arch): + self.assertEqual( + rowcol_tensor_quant_default_tile(arch), + dict(ROWCOL_TENSOR_QUANT_DEFAULT_TILE), + ) + + def test_gfx9_parts_take_the_mfma_tile(self): + for arch in ("gfx942", "gfx950", "gfx942:sramecc+:xnack-", ""): + with self.subTest(arch=arch): + self.assertEqual( + rowcol_tensor_quant_default_tile(arch), + dict(ROWCOL_TENSOR_QUANT_DEFAULT_TILE), + ) + + def test_returns_a_copy_not_the_shared_dict(self): + t = rowcol_tensor_quant_default_tile("gfx1250") + t["tile_m"] = -1 + self.assertNotEqual(ROWCOL_TENSOR_QUANT_DEFAULT_TILE_GFX1250["tile_m"], -1) + + if __name__ == "__main__": unittest.main() diff --git a/projects/composablekernel/dispatcher/tests/test_grouped_gemm_rowcolquant_utils.py b/projects/composablekernel/dispatcher/tests/test_grouped_gemm_rowcolquant_utils.py index cc19453b01c7..33c9fa2cc224 100644 --- a/projects/composablekernel/dispatcher/tests/test_grouped_gemm_rowcolquant_utils.py +++ b/projects/composablekernel/dispatcher/tests/test_grouped_gemm_rowcolquant_utils.py @@ -16,9 +16,12 @@ import sys from pathlib import Path +import pytest + sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "python")) +import grouped_gemm_rowcolquant_utils as UTILS from grouped_gemm_rowcolquant_utils import ( RowColQuantKernelConfig, RowColQuantGemmProblem, @@ -320,3 +323,56 @@ def test_default_bf8_config_name_in_codegen_defaults(self): f"default_bf8_config().name '{cfg.name}' is not produced by _default_config() " f"in the codegen. The two defaults have drifted — update one to match the other." ) + + +# ============================================================================= +# Arch-dependent hipcc defines +# ============================================================================= + + +class TestArchDefines: + """The OCP-FP8 define is deliberately a gfx12 *family* test. + + gfx1200/gfx1201/gfx1250 all use OCP FP8 encoding, so narrowing this to an + exact gfx1250 match would be wrong. This is the opposite of the tile + selector in codegen_common.rowcol_tensor_quant_default_tile(), which must be + exact because gfx1200/gfx1201 have a different 8-bit warp fragment. Both + behaviours are pinned so neither gets "fixed" into the other. + """ + + def _compile_argv(self, monkeypatch, tmp_path, gfx_arch): + captured = [] + + class _Result: + returncode = 0 + stdout = "" + stderr = "" + + def fake_run(cmd, *a, **kw): + captured.append(list(cmd)) + return _Result() + + monkeypatch.setattr(UTILS.subprocess, "run", fake_run) + ok = UTILS._compile_rowcolquant_kernel( + hpp_path=tmp_path / "k.hpp", + so_path=tmp_path / "k.so", + gfx_arch=gfx_arch, + ) + assert ok, "compile helper should report success when hipcc succeeds" + return captured[0] + + @pytest.mark.parametrize("arch", ["gfx1200", "gfx1201", "gfx1250", "gfx1250:xnack-"]) + def test_all_gfx12_parts_get_ocp_fp8(self, monkeypatch, tmp_path, arch): + argv = self._compile_argv(monkeypatch, tmp_path, arch) + assert "-DCK_USE_OCP_FP8" in argv + assert "-DCK_TILE_USE_OCP_FP8" in argv + + def test_gfx950_gets_ocp_fp8_and_mx(self, monkeypatch, tmp_path): + argv = self._compile_argv(monkeypatch, tmp_path, "gfx950") + assert "-DCK_USE_OCP_FP8" in argv + assert "-DCK_USE_NATIVE_MX_SUPPORT" in argv + + def test_gfx942_gets_neither(self, monkeypatch, tmp_path): + argv = self._compile_argv(monkeypatch, tmp_path, "gfx942:sramecc+:xnack-") + assert "-DCK_USE_OCP_FP8" not in argv + assert "-DCK_USE_NATIVE_MX_SUPPORT" not in argv diff --git a/projects/composablekernel/dispatcher/tests/test_grouped_gemm_tensorquant_utils.py b/projects/composablekernel/dispatcher/tests/test_grouped_gemm_tensorquant_utils.py index 381f796af92a..e92a65dcb8b8 100644 --- a/projects/composablekernel/dispatcher/tests/test_grouped_gemm_tensorquant_utils.py +++ b/projects/composablekernel/dispatcher/tests/test_grouped_gemm_tensorquant_utils.py @@ -16,9 +16,12 @@ import sys from pathlib import Path +import pytest + sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "python")) +import grouped_gemm_tensorquant_utils as UTILS from grouped_gemm_tensorquant_utils import ( TensorQuantKernelConfig, TensorQuantGemmProblem, @@ -327,3 +330,56 @@ def test_default_bf8_config_name_in_codegen_defaults(self): f"default_bf8_config().name '{cfg.name}' is not produced by _default_config() " f"in the codegen. The two defaults have drifted — update one to match the other." ) + + +# ============================================================================= +# Arch-dependent hipcc defines +# ============================================================================= + + +class TestArchDefines: + """The OCP-FP8 define is deliberately a gfx12 *family* test. + + gfx1200/gfx1201/gfx1250 all use OCP FP8 encoding, so narrowing this to an + exact gfx1250 match would be wrong. This is the opposite of the tile + selector in codegen_common.rowcol_tensor_quant_default_tile(), which must be + exact because gfx1200/gfx1201 have a different 8-bit warp fragment. Both + behaviours are pinned so neither gets "fixed" into the other. + """ + + def _compile_argv(self, monkeypatch, tmp_path, gfx_arch): + captured = [] + + class _Result: + returncode = 0 + stdout = "" + stderr = "" + + def fake_run(cmd, *a, **kw): + captured.append(list(cmd)) + return _Result() + + monkeypatch.setattr(UTILS.subprocess, "run", fake_run) + ok = UTILS._compile_tensorquant_kernel( + hpp_path=tmp_path / "k.hpp", + so_path=tmp_path / "k.so", + gfx_arch=gfx_arch, + ) + assert ok, "compile helper should report success when hipcc succeeds" + return captured[0] + + @pytest.mark.parametrize("arch", ["gfx1200", "gfx1201", "gfx1250", "gfx1250:xnack-"]) + def test_all_gfx12_parts_get_ocp_fp8(self, monkeypatch, tmp_path, arch): + argv = self._compile_argv(monkeypatch, tmp_path, arch) + assert "-DCK_USE_OCP_FP8" in argv + assert "-DCK_TILE_USE_OCP_FP8" in argv + + def test_gfx950_gets_ocp_fp8_and_mx(self, monkeypatch, tmp_path): + argv = self._compile_argv(monkeypatch, tmp_path, "gfx950") + assert "-DCK_USE_OCP_FP8" in argv + assert "-DCK_USE_NATIVE_MX_SUPPORT" in argv + + def test_gfx942_gets_neither(self, monkeypatch, tmp_path): + argv = self._compile_argv(monkeypatch, tmp_path, "gfx942:sramecc+:xnack-") + assert "-DCK_USE_OCP_FP8" not in argv + assert "-DCK_USE_NATIVE_MX_SUPPORT" not in argv diff --git a/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py index 65ec88e93f05..06fe3402907e 100644 --- a/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py @@ -31,7 +31,11 @@ import pytest sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "python")) +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "codegen")) +# normalize_gfx_arch lives in codegen_common: single source of truth shared by +# the codegen tile selector, the runtime helpers and these tests. +from codegen_common import normalize_gfx_arch # noqa: E402 from grouped_gemm_rowcolquant_utils import ( RowColQuantGemmProblem, RowColQuantGpuGemmRunner, @@ -53,17 +57,6 @@ def _has_hipcc() -> bool: return shutil.which("hipcc") is not None -def normalize_gfx_arch(arch: str) -> str: - """Strip feature suffixes from a gfx target string. - - ``rocm_agent_enumerator`` and ``hipDeviceProp_t::gcnArchName`` may report the - target with trailing feature flags, e.g. ``"gfx942:sramecc+:xnack-"``. Every - comparison we do here (and the ``--offload-arch`` we hand to hipcc) wants the - bare target, so normalize once at the boundary. - """ - return arch.split(":", 1)[0] - - def arch_is_supported(arch: str, supported=None) -> bool: """True if `arch` names a supported target, ignoring feature suffixes. diff --git a/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py index b4e5a5b67725..155a8e7cac3b 100644 --- a/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py @@ -31,7 +31,11 @@ import pytest sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "python")) +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "codegen")) +# normalize_gfx_arch lives in codegen_common: single source of truth shared by +# the codegen tile selector, the runtime helpers and these tests. +from codegen_common import normalize_gfx_arch # noqa: E402 from grouped_gemm_tensorquant_utils import ( TensorQuantGemmProblem, TensorQuantGpuGemmRunner, @@ -53,17 +57,6 @@ def _has_hipcc() -> bool: return shutil.which("hipcc") is not None -def normalize_gfx_arch(arch: str) -> str: - """Strip feature suffixes from a gfx target string. - - ``rocm_agent_enumerator`` and ``hipDeviceProp_t::gcnArchName`` may report the - target with trailing feature flags, e.g. ``"gfx942:sramecc+:xnack-"``. Every - comparison we do here (and the ``--offload-arch`` we hand to hipcc) wants the - bare target, so normalize once at the boundary. - """ - return arch.split(":", 1)[0] - - def arch_is_supported(arch: str, supported=None) -> bool: """True if `arch` names a supported target, ignoring feature suffixes. From c17151fd58452ec8b170ce674e149fc7468b9d9e Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk <3836908+ozturkosu@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:07:11 -0400 Subject: [PATCH 04/24] fix(ck-tile): version the .so cache key and normalize caller-supplied arch The grouped RowColQuant/TensorQuant .so cache was keyed on (kernel name, arch), which says nothing about the exported symbol set. A caller with a persistent output_dir predating the dispatcher_get_tile_n()/dispatcher_get_pad_n() exports was handed that stale artifact, and the runner then died at attribute lookup with a bare "undefined symbol". Default usage is tempfile.mkdtemp() so CI never saw it, but the validation harnesses use persistent directories precisely to get cache hits. Fold an _SO_ABI revision into the artifact filename so a pre-ABI .so simply cannot be selected -- rejecting the stale artifact rather than masking it with a symbol-lookup fallback, which would hide the staleness instead of ending it. Reproduced on gfx1250 against a real pre-ABI library built from the parent commit's ctypes source: it raises "undefined symbol: dispatcher_get_tile_n" under the old name and is ignored under the new one. normalize_gfx_arch() was applied only to the detection branch, so a caller passing gfx_arch="gfx1250:xnack-" sent the suffixed string straight into --offload-arch, -DGFX_ARCH and the cache name. Normalize once at each entry boundary instead: the setup function, the compile helper, and _default_config (reached from the public default_fp8_config/default_bf8_config). The GPU correctness tests derived the host fp8 encoding from the module-level detected arch rather than the arch being compiled for, so with --gfx pointing at a different target the host bytes and the kernel disagreed and decoded to NaN/Inf on device. Thread the arch through _fp8_ml_dtype/_encode_fp8/_decode_fp8 and _make_inputs. The gfx12-family test inside _fp8_uses_ocp is correct as-is (gfx1200/gfx1201 use OCP FP8 too) and is unchanged. Register test_codegen_common.py and test_gpu_test_arch_gate.py with CTest; both existed but nothing ran them. Validated on gfx1250: 4/4 kernels compile with both exports present, worst global max_rel 4.22e-4 across rowcolquant/tensorquant x fp8/bf8 over five shapes each, M%4!=0 cleanly rejected for RowColQuant and correct for TensorQuant, reported tile_n=64 matches the compiled kernel, and a suffixed target builds and runs end to end. Dispatcher suite 212 passed / 0 failed on device. --- .../python/grouped_gemm_rowcolquant_utils.py | 41 ++++++- .../python/grouped_gemm_tensorquant_utils.py | 41 ++++++- .../dispatcher/tests/CMakeLists.txt | 31 +++++ .../test_grouped_gemm_rowcolquant_utils.py | 114 ++++++++++++++++++ .../test_grouped_gemm_tensorquant_utils.py | 114 ++++++++++++++++++ .../tests/test_rowcolquant_gpu_correctness.py | 43 ++++--- .../tests/test_tensorquant_gpu_correctness.py | 43 ++++--- 7 files changed, 389 insertions(+), 38 deletions(-) diff --git a/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py b/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py index ec469b25e8c5..c6c6153199be 100644 --- a/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py +++ b/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py @@ -58,6 +58,23 @@ _DEFAULT_HIPCC = "hipcc" _DEFAULT_GFX_ARCH = "gfx950" +# ABI revision of the compiled .so, folded into the artifact filename. +# +# setup_multiple_rowcolquant_dispatchers() reuses an existing .so when the name +# matches, and callers that pass a persistent output_dir (the validation +# harnesses do, precisely to get cache hits) can therefore be handed an artifact +# built by an older revision of this module. The name used to be keyed on +# (kernel name, arch) only, which does not describe the exported symbol set, so +# a .so predating the dispatcher_get_tile_n()/dispatcher_get_pad_n() exports +# would be selected and then fail at attribute-lookup time with a bare +# "undefined symbol". Bump this whenever the exported C ABI of +# bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp changes; the new name +# simply cannot collide with the stale artifact, which is then ignored rather +# than papered over with a runtime fallback. +# 1 -> original export set +# 2 -> added dispatcher_get_tile_n() / dispatcher_get_pad_n() +_SO_ABI = 2 + # ============================================================================= # RowColQuantKernelConfig — byte-exact naming with codegen @@ -562,6 +579,12 @@ def _compile_rowcolquant_kernel( extra_include_dirs: Optional[List[str]] = None, ) -> bool: """Compile a generated .hpp into a .so via hipcc (compile then link).""" + # Normalize once, here at the boundary, so every downstream use -- the arch + # defines below and the --offload-arch/-DGFX_ARCH we hand to hipcc -- sees the + # bare target. A caller-supplied "gfx1250:xnack-" must not reach the compiler + # flags. + gfx_arch = normalize_gfx_arch(gfx_arch) + ck_include = _get_ck_include_dir() static_lib = _get_dispatcher_static_lib() @@ -572,10 +595,9 @@ def _compile_rowcolquant_kernel( # FP8 encoding, so an exact-gfx1250 test here would be WRONG. This is the # opposite of the tile selector in codegen_common.rowcol_tensor_quant_default_tile(), # which must be exact because gfx1200/gfx1201 have a different 8-bit warp fragment. - _arch_base = normalize_gfx_arch(gfx_arch) - if _arch_base.startswith("gfx12") or _arch_base == "gfx950": + if gfx_arch.startswith("gfx12") or gfx_arch == "gfx950": arch_defines += ["-DCK_USE_OCP_FP8", "-DCK_TILE_USE_OCP_FP8"] - if _arch_base == "gfx950": + if gfx_arch == "gfx950": arch_defines += ["-DCK_USE_NATIVE_MX_SUPPORT", "-DCK_GFX950_SUPPORT"] compile_cmd = [hipcc, "-c", "-fPIC", "-O3", "-std=c++17", @@ -655,7 +677,10 @@ def setup_multiple_rowcolquant_dispatchers( if not configs: return [] - arch = gfx_arch or _detect_gpu_arch() + # Normalize the explicit branch too, not just detection: an explicitly passed + # "gfx1250:xnack-" would otherwise flow into --offload-arch, -DGFX_ARCH and the + # .so cache name. _detect_gpu_arch() already normalizes. + arch = normalize_gfx_arch(gfx_arch) if gfx_arch else _detect_gpu_arch() base_dir = output_dir or Path(tempfile.mkdtemp(prefix="rowcolquant_dispatcher_")) base_dir.mkdir(parents=True, exist_ok=True) @@ -680,7 +705,9 @@ def _build_one(idx: int, cfg: RowColQuantKernelConfig) -> Tuple[int, Optional[Pa if hpp is None: return idx, None - so = so_dir / f"lib{cfg.name}_{arch}.so" + # Name carries the ABI revision: a pre-_SO_ABI artifact in a persistent + # output_dir can never be selected here (see _SO_ABI). + so = so_dir / f"lib{cfg.name}_{arch}_abi{_SO_ABI}.so" if so.exists(): log.info(" [cached] %s", so.name) return idx, so @@ -741,6 +768,10 @@ def _default_config(dtype: str, gfx_arch: str) -> RowColQuantKernelConfig: four pad/persistent flags) is forwarded from the shared dict; only block_size and k_block_per_cu are codegen-only and are left to the dataclass defaults. """ + # Normalize at this boundary too: default_fp8_config()/default_bf8_config() are + # public entry points, so a caller-supplied "gfx1250:xnack-" must not be stored + # on the config and observed by later consumers. + gfx_arch = normalize_gfx_arch(gfx_arch) traits = ROWCOL_TENSOR_QUANT_DEFAULT_TRAITS return RowColQuantKernelConfig( dtype=dtype, diff --git a/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py b/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py index a9f42aecc833..fc459ce41034 100644 --- a/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py +++ b/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py @@ -58,6 +58,23 @@ _DEFAULT_HIPCC = "hipcc" _DEFAULT_GFX_ARCH = "gfx950" +# ABI revision of the compiled .so, folded into the artifact filename. +# +# setup_multiple_tensorquant_dispatchers() reuses an existing .so when the name +# matches, and callers that pass a persistent output_dir (the validation +# harnesses do, precisely to get cache hits) can therefore be handed an artifact +# built by an older revision of this module. The name used to be keyed on +# (kernel name, arch) only, which does not describe the exported symbol set, so +# a .so predating the dispatcher_get_tile_n()/dispatcher_get_pad_n() exports +# would be selected and then fail at attribute-lookup time with a bare +# "undefined symbol". Bump this whenever the exported C ABI of +# bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp changes; the new name +# simply cannot collide with the stale artifact, which is then ignored rather +# than papered over with a runtime fallback. +# 1 -> original export set +# 2 -> added dispatcher_get_tile_n() / dispatcher_get_pad_n() +_SO_ABI = 2 + # ============================================================================= # TensorQuantKernelConfig — byte-exact naming with codegen @@ -547,6 +564,12 @@ def _compile_tensorquant_kernel( extra_include_dirs: Optional[List[str]] = None, ) -> bool: """Compile a generated .hpp into a .so via hipcc (compile then link).""" + # Normalize once, here at the boundary, so every downstream use -- the arch + # defines below and the --offload-arch/-DGFX_ARCH we hand to hipcc -- sees the + # bare target. A caller-supplied "gfx1250:xnack-" must not reach the compiler + # flags. + gfx_arch = normalize_gfx_arch(gfx_arch) + ck_include = _get_ck_include_dir() static_lib = _get_dispatcher_static_lib() @@ -557,10 +580,9 @@ def _compile_tensorquant_kernel( # FP8 encoding, so an exact-gfx1250 test here would be WRONG. This is the # opposite of the tile selector in codegen_common.rowcol_tensor_quant_default_tile(), # which must be exact because gfx1200/gfx1201 have a different 8-bit warp fragment. - _arch_base = normalize_gfx_arch(gfx_arch) - if _arch_base.startswith("gfx12") or _arch_base == "gfx950": + if gfx_arch.startswith("gfx12") or gfx_arch == "gfx950": arch_defines += ["-DCK_USE_OCP_FP8", "-DCK_TILE_USE_OCP_FP8"] - if _arch_base == "gfx950": + if gfx_arch == "gfx950": arch_defines += ["-DCK_USE_NATIVE_MX_SUPPORT", "-DCK_GFX950_SUPPORT"] compile_cmd = [hipcc, "-c", "-fPIC", "-O3", "-std=c++17", @@ -640,7 +662,10 @@ def setup_multiple_tensorquant_dispatchers( if not configs: return [] - arch = gfx_arch or _detect_gpu_arch() + # Normalize the explicit branch too, not just detection: an explicitly passed + # "gfx1250:xnack-" would otherwise flow into --offload-arch, -DGFX_ARCH and the + # .so cache name. _detect_gpu_arch() already normalizes. + arch = normalize_gfx_arch(gfx_arch) if gfx_arch else _detect_gpu_arch() base_dir = output_dir or Path(tempfile.mkdtemp(prefix="tensorquant_dispatcher_")) base_dir.mkdir(parents=True, exist_ok=True) @@ -665,7 +690,9 @@ def _build_one(idx: int, cfg: TensorQuantKernelConfig) -> Tuple[int, Optional[Pa if hpp is None: return idx, None - so = so_dir / f"lib{cfg.name}_{arch}.so" + # Name carries the ABI revision: a pre-_SO_ABI artifact in a persistent + # output_dir can never be selected here (see _SO_ABI). + so = so_dir / f"lib{cfg.name}_{arch}_abi{_SO_ABI}.so" if so.exists(): log.info(" [cached] %s", so.name) return idx, so @@ -726,6 +753,10 @@ def _default_config(dtype: str, gfx_arch: str) -> TensorQuantKernelConfig: four pad/persistent flags) is forwarded from the shared dict; only block_size and k_block_per_cu are codegen-only and are left to the dataclass defaults. """ + # Normalize at this boundary too: default_fp8_config()/default_bf8_config() are + # public entry points, so a caller-supplied "gfx1250:xnack-" must not be stored + # on the config and observed by later consumers. + gfx_arch = normalize_gfx_arch(gfx_arch) traits = ROWCOL_TENSOR_QUANT_DEFAULT_TRAITS return TensorQuantKernelConfig( dtype=dtype, diff --git a/projects/composablekernel/dispatcher/tests/CMakeLists.txt b/projects/composablekernel/dispatcher/tests/CMakeLists.txt index 316c3d373e56..12c9b61eb008 100644 --- a/projects/composablekernel/dispatcher/tests/CMakeLists.txt +++ b/projects/composablekernel/dispatcher/tests/CMakeLists.txt @@ -114,6 +114,37 @@ set_tests_properties(dispatcher_test_arch_filter_constraints PROPERTIES ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" ) +# codegen_common shared-infrastructure regression test. Covers normalize_gfx_arch() +# and the EXACT-gfx1250 RowColQuant/TensorQuant tile selector: a family-wide match +# there would hand gfx1200/gfx1201 a tile that compiles cleanly and returns garbage, +# which no downstream check catches. +add_test( + NAME dispatcher_test_codegen_common + COMMAND ${Python3_EXECUTABLE} -m pytest -q ${CMAKE_CURRENT_SOURCE_DIR}/test_codegen_common.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) + +set_tests_properties(dispatcher_test_codegen_common PROPERTIES + LABELS "dispatcher;python;codegen;validation" + TIMEOUT 120 + ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" +) + +# Arch-gate predicate used by the RowColQuant/TensorQuant GPU tests. CPU-only: it +# exercises the string predicate, not a device. Worth its own registration because +# the failure mode it guards is a SKIP -- CI stays green while covering nothing. +add_test( + NAME dispatcher_test_gpu_test_arch_gate + COMMAND ${Python3_EXECUTABLE} -m pytest -q ${CMAKE_CURRENT_SOURCE_DIR}/test_gpu_test_arch_gate.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) + +set_tests_properties(dispatcher_test_gpu_test_arch_gate PROPERTIES + LABELS "dispatcher;python;validation" + TIMEOUT 60 + ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" +) + # gemm_utils host-helper + runner-shape regression test. Also acts as an import # canary: a merge that truncates python/gemm_utils.py (see #9308) makes this fail. add_test( diff --git a/projects/composablekernel/dispatcher/tests/test_grouped_gemm_rowcolquant_utils.py b/projects/composablekernel/dispatcher/tests/test_grouped_gemm_rowcolquant_utils.py index 33c9fa2cc224..f30e6d34aa04 100644 --- a/projects/composablekernel/dispatcher/tests/test_grouped_gemm_rowcolquant_utils.py +++ b/projects/composablekernel/dispatcher/tests/test_grouped_gemm_rowcolquant_utils.py @@ -376,3 +376,117 @@ def test_gfx942_gets_neither(self, monkeypatch, tmp_path): argv = self._compile_argv(monkeypatch, tmp_path, "gfx942:sramecc+:xnack-") assert "-DCK_USE_OCP_FP8" not in argv assert "-DCK_USE_NATIVE_MX_SUPPORT" not in argv + + +# ============================================================================= +# Compiler-flag arch normalization +# ============================================================================= + + +class TestArchNormalizationInCompileFlags: + """A suffixed target must never reach --offload-arch / -DGFX_ARCH. + + normalize_gfx_arch() used to be applied only to the *detected* arch. A caller + passing gfx_arch="gfx1250:xnack-" therefore had the raw string forwarded into + the compiler flags. Normalization now happens once at the entry boundary. + """ + + def _compile_argv(self, monkeypatch, tmp_path, gfx_arch): + captured = [] + + class _Result: + returncode = 0 + stdout = "" + stderr = "" + + def fake_run(cmd, *a, **kw): + captured.append(cmd) + return _Result() + + monkeypatch.setattr(UTILS.subprocess, "run", fake_run) + ok = UTILS._compile_rowcolquant_kernel( + hpp_path=tmp_path / "k.hpp", + so_path=tmp_path / "k.so", + gfx_arch=gfx_arch, + ) + assert ok + return captured[0] + + def test_suffixed_arch_is_stripped_from_offload_arch(self, monkeypatch, tmp_path): + argv = self._compile_argv(monkeypatch, tmp_path, "gfx1250:xnack-") + assert "--offload-arch=gfx1250" in argv + assert not any("xnack" in tok for tok in argv), argv + + def test_suffixed_arch_is_stripped_from_gfx_arch_define(self, monkeypatch, tmp_path): + argv = self._compile_argv(monkeypatch, tmp_path, "gfx942:sramecc+:xnack-") + assert '-DGFX_ARCH="gfx942"' in argv + + def test_default_config_stores_bare_arch(self): + assert default_fp8_config(gfx_arch="gfx1250:xnack-").gfx_arch == "gfx1250" + assert default_bf8_config(gfx_arch="gfx942:sramecc+").gfx_arch == "gfx942" + + +# ============================================================================= +# .so cache key encodes the ABI +# ============================================================================= + + +class TestSoCacheAbiKey: + """A pre-ABI artifact in a persistent output_dir must not be reused. + + setup_multiple_rowcolquant_dispatchers() reuses a .so when the filename + matches. The name used to be lib{kernel}_{arch}.so, which says nothing about + the exported symbol set, so a .so predating the + dispatcher_get_tile_n()/dispatcher_get_pad_n() exports was selected and then + died at attribute lookup with "undefined symbol". The name now carries + _SO_ABI, so the stale artifact simply cannot be selected. + """ + + def test_abi_is_versioned(self): + assert isinstance(UTILS._SO_ABI, int) and UTILS._SO_ABI >= 2 + + def test_pre_abi_artifact_is_not_reused(self, monkeypatch, tmp_path): + cfg = default_fp8_config(gfx_arch="gfx950") + + so_dir = tmp_path / "libs" + so_dir.mkdir(parents=True) + # Exactly the name the old code would have produced and reused. + stale = so_dir / f"lib{cfg.name}_gfx950.so" + stale.write_bytes(b"stale pre-ABI artifact") + + compiled = [] + + def fake_compile(hpp_path, so_path, gfx_arch, **kw): + compiled.append(so_path) + so_path.write_bytes(b"fresh") + return True + + monkeypatch.setattr(UTILS, "_compile_rowcolquant_kernel", fake_compile) + + out = UTILS.setup_multiple_rowcolquant_dispatchers( + configs=[cfg], output_dir=tmp_path, gfx_arch="gfx950", parallel=False, + ) + + assert compiled, "stale pre-ABI .so was reused instead of rebuilt" + assert out[0] != stale + assert f"_abi{UTILS._SO_ABI}.so" in out[0].name + assert stale.read_bytes() == b"stale pre-ABI artifact", "stale file was clobbered" + + def test_second_call_hits_the_cache(self, monkeypatch, tmp_path): + cfg = default_fp8_config(gfx_arch="gfx950") + calls = [] + + def fake_compile(hpp_path, so_path, gfx_arch, **kw): + calls.append(so_path) + so_path.write_bytes(b"fresh") + return True + + monkeypatch.setattr(UTILS, "_compile_rowcolquant_kernel", fake_compile) + + first = UTILS.setup_multiple_rowcolquant_dispatchers( + configs=[cfg], output_dir=tmp_path, gfx_arch="gfx950", parallel=False) + second = UTILS.setup_multiple_rowcolquant_dispatchers( + configs=[cfg], output_dir=tmp_path, gfx_arch="gfx950", parallel=False) + + assert len(calls) == 1, "second call should have hit the cache" + assert first[0] == second[0] diff --git a/projects/composablekernel/dispatcher/tests/test_grouped_gemm_tensorquant_utils.py b/projects/composablekernel/dispatcher/tests/test_grouped_gemm_tensorquant_utils.py index e92a65dcb8b8..c6aa22b4fcfa 100644 --- a/projects/composablekernel/dispatcher/tests/test_grouped_gemm_tensorquant_utils.py +++ b/projects/composablekernel/dispatcher/tests/test_grouped_gemm_tensorquant_utils.py @@ -383,3 +383,117 @@ def test_gfx942_gets_neither(self, monkeypatch, tmp_path): argv = self._compile_argv(monkeypatch, tmp_path, "gfx942:sramecc+:xnack-") assert "-DCK_USE_OCP_FP8" not in argv assert "-DCK_USE_NATIVE_MX_SUPPORT" not in argv + + +# ============================================================================= +# Compiler-flag arch normalization +# ============================================================================= + + +class TestArchNormalizationInCompileFlags: + """A suffixed target must never reach --offload-arch / -DGFX_ARCH. + + normalize_gfx_arch() used to be applied only to the *detected* arch. A caller + passing gfx_arch="gfx1250:xnack-" therefore had the raw string forwarded into + the compiler flags. Normalization now happens once at the entry boundary. + """ + + def _compile_argv(self, monkeypatch, tmp_path, gfx_arch): + captured = [] + + class _Result: + returncode = 0 + stdout = "" + stderr = "" + + def fake_run(cmd, *a, **kw): + captured.append(cmd) + return _Result() + + monkeypatch.setattr(UTILS.subprocess, "run", fake_run) + ok = UTILS._compile_tensorquant_kernel( + hpp_path=tmp_path / "k.hpp", + so_path=tmp_path / "k.so", + gfx_arch=gfx_arch, + ) + assert ok + return captured[0] + + def test_suffixed_arch_is_stripped_from_offload_arch(self, monkeypatch, tmp_path): + argv = self._compile_argv(monkeypatch, tmp_path, "gfx1250:xnack-") + assert "--offload-arch=gfx1250" in argv + assert not any("xnack" in tok for tok in argv), argv + + def test_suffixed_arch_is_stripped_from_gfx_arch_define(self, monkeypatch, tmp_path): + argv = self._compile_argv(monkeypatch, tmp_path, "gfx942:sramecc+:xnack-") + assert '-DGFX_ARCH="gfx942"' in argv + + def test_default_config_stores_bare_arch(self): + assert default_fp8_config(gfx_arch="gfx1250:xnack-").gfx_arch == "gfx1250" + assert default_bf8_config(gfx_arch="gfx942:sramecc+").gfx_arch == "gfx942" + + +# ============================================================================= +# .so cache key encodes the ABI +# ============================================================================= + + +class TestSoCacheAbiKey: + """A pre-ABI artifact in a persistent output_dir must not be reused. + + setup_multiple_tensorquant_dispatchers() reuses a .so when the filename + matches. The name used to be lib{kernel}_{arch}.so, which says nothing about + the exported symbol set, so a .so predating the + dispatcher_get_tile_n()/dispatcher_get_pad_n() exports was selected and then + died at attribute lookup with "undefined symbol". The name now carries + _SO_ABI, so the stale artifact simply cannot be selected. + """ + + def test_abi_is_versioned(self): + assert isinstance(UTILS._SO_ABI, int) and UTILS._SO_ABI >= 2 + + def test_pre_abi_artifact_is_not_reused(self, monkeypatch, tmp_path): + cfg = default_fp8_config(gfx_arch="gfx950") + + so_dir = tmp_path / "libs" + so_dir.mkdir(parents=True) + # Exactly the name the old code would have produced and reused. + stale = so_dir / f"lib{cfg.name}_gfx950.so" + stale.write_bytes(b"stale pre-ABI artifact") + + compiled = [] + + def fake_compile(hpp_path, so_path, gfx_arch, **kw): + compiled.append(so_path) + so_path.write_bytes(b"fresh") + return True + + monkeypatch.setattr(UTILS, "_compile_tensorquant_kernel", fake_compile) + + out = UTILS.setup_multiple_tensorquant_dispatchers( + configs=[cfg], output_dir=tmp_path, gfx_arch="gfx950", parallel=False, + ) + + assert compiled, "stale pre-ABI .so was reused instead of rebuilt" + assert out[0] != stale + assert f"_abi{UTILS._SO_ABI}.so" in out[0].name + assert stale.read_bytes() == b"stale pre-ABI artifact", "stale file was clobbered" + + def test_second_call_hits_the_cache(self, monkeypatch, tmp_path): + cfg = default_fp8_config(gfx_arch="gfx950") + calls = [] + + def fake_compile(hpp_path, so_path, gfx_arch, **kw): + calls.append(so_path) + so_path.write_bytes(b"fresh") + return True + + monkeypatch.setattr(UTILS, "_compile_tensorquant_kernel", fake_compile) + + first = UTILS.setup_multiple_tensorquant_dispatchers( + configs=[cfg], output_dir=tmp_path, gfx_arch="gfx950", parallel=False) + second = UTILS.setup_multiple_tensorquant_dispatchers( + configs=[cfg], output_dir=tmp_path, gfx_arch="gfx950", parallel=False) + + assert len(calls) == 1, "second call should have hit the cache" + assert first[0] == second[0] diff --git a/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py index 06fe3402907e..af915d4d44db 100644 --- a/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py @@ -129,26 +129,33 @@ def _fp8_uses_ocp(arch: str) -> bool: return "gfx950" in arch or "gfx12" in arch -def _fp8_ml_dtype(dtype: str): - """Return the ml_dtypes fp8 type matching the compiled kernel's arch.""" +def _fp8_ml_dtype(dtype: str, arch: str): + """Return the ml_dtypes fp8 type matching the compiled kernel's arch. + + `arch` is the arch the kernel is being COMPILED for, which is not necessarily + the arch this host detects: main() takes --gfx. Reading the module-level + detected arch here would silently encode the host bytes in the other format, + and the kernel would decode them to NaN/Inf on device. So it is a parameter, + not a global. + """ _require_ml_dtypes() - if _fp8_uses_ocp(_GFX_ARCH): + if _fp8_uses_ocp(arch): return _ml_dtypes.float8_e4m3fn if dtype == "fp8" else _ml_dtypes.float8_e5m2 return _ml_dtypes.float8_e4m3fnuz if dtype == "fp8" else _ml_dtypes.float8_e5m2fnuz -def _encode_fp8(arr: np.ndarray, dtype: str) -> np.ndarray: +def _encode_fp8(arr: np.ndarray, dtype: str, arch: str) -> np.ndarray: """Encode float32 → fp8/bf8 bytes (uint8 view). Requires ml_dtypes. The fp8 format (OCP vs FNUZ) follows the compiled kernel's arch; see _fp8_uses_ocp. """ - ml_t = _fp8_ml_dtype(dtype) + ml_t = _fp8_ml_dtype(dtype, arch) return arr.astype(ml_t).view(np.uint8) -def _decode_fp8(arr: np.ndarray, dtype: str) -> np.ndarray: +def _decode_fp8(arr: np.ndarray, dtype: str, arch: str) -> np.ndarray: """Decode fp8/bf8 bytes (uint8 view) → float32. Requires ml_dtypes.""" - ml_t = _fp8_ml_dtype(dtype) + ml_t = _fp8_ml_dtype(dtype, arch) return arr.view(ml_t).astype(np.float32) @@ -226,16 +233,22 @@ def _run_one(label: str, config, M: int, N: int, K: int, # Input factory # --------------------------------------------------------------------------- -def _make_inputs(M, N, K, dtype="fp8", seed=42): +def _make_inputs(M, N, K, dtype="fp8", seed=42, arch=None): + """Build host inputs. `arch` must be the arch the kernel is compiled for. + + Defaults to the detected arch, which is right for the pytest cases (they + compile for _GFX_ARCH); main() passes its --gfx value explicitly. + """ + arch = _GFX_ARCH if arch is None else arch rng = np.random.default_rng(seed) A_f32 = rng.uniform(-1.0, 1.0, (M, K)).astype(np.float32) B_f32 = rng.uniform(-1.0, 1.0, (K, N)).astype(np.float32) AQ = rng.uniform(0.5, 2.0, (M,)).astype(np.float32) # per-row A scale BQ = rng.uniform(0.5, 2.0, (N,)).astype(np.float32) # per-col B scale - A_raw = _encode_fp8(A_f32, dtype) - B_raw = _encode_fp8(B_f32, dtype) - A_dec = _decode_fp8(A_raw, dtype) - B_dec = _decode_fp8(B_raw, dtype) + A_raw = _encode_fp8(A_f32, dtype, arch) + B_raw = _encode_fp8(B_f32, dtype, arch) + A_dec = _decode_fp8(A_raw, dtype, arch) + B_dec = _decode_fp8(B_raw, dtype, arch) return A_raw, A_dec, B_raw, B_dec, AQ, BQ @@ -305,12 +318,14 @@ def test_rowcolquant_timing_positive(tmp_path): # --------------------------------------------------------------------------- TESTS = [ + # arch=gfx, not the detected arch: with --gfx the host encoding must follow the + # arch the kernel is compiled for (see _fp8_ml_dtype). ("C4/fp8", lambda od, gfx: _run_one( "C4/fp8", default_fp8_config(gfx_arch=gfx), 128, 128, 192, - *_make_inputs(128, 128, 192, "fp8"), Path(od), gfx_arch=gfx)), + *_make_inputs(128, 128, 192, "fp8", arch=gfx), Path(od), gfx_arch=gfx)), ("C4/bf8", lambda od, gfx: _run_one( "C4/bf8", default_bf8_config(gfx_arch=gfx), 128, 128, 192, - *_make_inputs(128, 128, 192, "bf8"), Path(od), gfx_arch=gfx)), + *_make_inputs(128, 128, 192, "bf8", arch=gfx), Path(od), gfx_arch=gfx)), ] diff --git a/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py index 155a8e7cac3b..da5c10b9b142 100644 --- a/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py @@ -129,26 +129,33 @@ def _fp8_uses_ocp(arch: str) -> bool: return "gfx950" in arch or "gfx12" in arch -def _fp8_ml_dtype(dtype: str): - """Return the ml_dtypes fp8 type matching the compiled kernel's arch.""" +def _fp8_ml_dtype(dtype: str, arch: str): + """Return the ml_dtypes fp8 type matching the compiled kernel's arch. + + `arch` is the arch the kernel is being COMPILED for, which is not necessarily + the arch this host detects: main() takes --gfx. Reading the module-level + detected arch here would silently encode the host bytes in the other format, + and the kernel would decode them to NaN/Inf on device. So it is a parameter, + not a global. + """ _require_ml_dtypes() - if _fp8_uses_ocp(_GFX_ARCH): + if _fp8_uses_ocp(arch): return _ml_dtypes.float8_e4m3fn if dtype == "fp8" else _ml_dtypes.float8_e5m2 return _ml_dtypes.float8_e4m3fnuz if dtype == "fp8" else _ml_dtypes.float8_e5m2fnuz -def _encode_fp8(arr: np.ndarray, dtype: str) -> np.ndarray: +def _encode_fp8(arr: np.ndarray, dtype: str, arch: str) -> np.ndarray: """Encode float32 → fp8/bf8 bytes (uint8 view). Requires ml_dtypes. The fp8 format (OCP vs FNUZ) follows the compiled kernel's arch; see _fp8_uses_ocp. """ - ml_t = _fp8_ml_dtype(dtype) + ml_t = _fp8_ml_dtype(dtype, arch) return arr.astype(ml_t).view(np.uint8) -def _decode_fp8(arr: np.ndarray, dtype: str) -> np.ndarray: +def _decode_fp8(arr: np.ndarray, dtype: str, arch: str) -> np.ndarray: """Decode fp8/bf8 bytes (uint8 view) → float32. Requires ml_dtypes.""" - ml_t = _fp8_ml_dtype(dtype) + ml_t = _fp8_ml_dtype(dtype, arch) return arr.view(ml_t).astype(np.float32) @@ -223,17 +230,23 @@ def _run_one(label: str, config, M: int, N: int, K: int, # Input factory # --------------------------------------------------------------------------- -def _make_inputs(M, N, K, dtype="fp8", seed=42): +def _make_inputs(M, N, K, dtype="fp8", seed=42, arch=None): + """Build host inputs. `arch` must be the arch the kernel is compiled for. + + Defaults to the detected arch, which is right for the pytest cases (they + compile for _GFX_ARCH); main() passes its --gfx value explicitly. + """ + arch = _GFX_ARCH if arch is None else arch rng = np.random.default_rng(seed) A_f32 = rng.uniform(-1.0, 1.0, (M, K)).astype(np.float32) B_f32 = rng.uniform(-1.0, 1.0, (K, N)).astype(np.float32) # Single per-tensor scales in (0.5, 2.0] AQ = rng.uniform(0.5, 2.0, (1,)).astype(np.float32) BQ = rng.uniform(0.5, 2.0, (1,)).astype(np.float32) - A_raw = _encode_fp8(A_f32, dtype) - B_raw = _encode_fp8(B_f32, dtype) - A_dec = _decode_fp8(A_raw, dtype) - B_dec = _decode_fp8(B_raw, dtype) + A_raw = _encode_fp8(A_f32, dtype, arch) + B_raw = _encode_fp8(B_f32, dtype, arch) + A_dec = _decode_fp8(A_raw, dtype, arch) + B_dec = _decode_fp8(B_raw, dtype, arch) return A_raw, A_dec, B_raw, B_dec, AQ, BQ @@ -303,12 +316,14 @@ def test_tensorquant_timing_positive(tmp_path): # --------------------------------------------------------------------------- TESTS = [ + # arch=gfx, not the detected arch: with --gfx the host encoding must follow the + # arch the kernel is compiled for (see _fp8_ml_dtype). ("C4/fp8", lambda od, gfx: _run_one( "C4/fp8", default_fp8_config(gfx_arch=gfx), 128, 128, 192, - *_make_inputs(128, 128, 192, "fp8"), Path(od), gfx_arch=gfx)), + *_make_inputs(128, 128, 192, "fp8", arch=gfx), Path(od), gfx_arch=gfx)), ("C4/bf8", lambda od, gfx: _run_one( "C4/bf8", default_bf8_config(gfx_arch=gfx), 128, 128, 192, - *_make_inputs(128, 128, 192, "bf8"), Path(od), gfx_arch=gfx)), + *_make_inputs(128, 128, 192, "bf8", arch=gfx), Path(od), gfx_arch=gfx)), ] From 6bbbcb4887a503c66954a0b673ffe809889876bd Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk <3836908+ozturkosu@users.noreply.github.com> Date: Wed, 9 Sep 2026 21:02:12 -0400 Subject: [PATCH 05/24] fix(ck-tile): reject gfx1250-invalid warp maps and warp tiles at validation The sweep validity tables were written for gfx9 and let two families of configuration through on gfx1250, both of which are worse than a build error. WARP_SUPPORTED_COMBINATIONS listed nine gfx1250 warp maps. Four of them are 8-warp blocks -- [2,4,1] [4,2,1] [1,8,1] [8,1,1] -- and gfx1250 is wave32, so a block of more than four warps has no launchable kernel entry: 3,220 of 3,220 such rows aborted at launch in a GPU sweep. A fifth, [1,2,2], compiles and then returns wrong results (GPU-measured max_rel 1.37). Only [1,4,1] [2,1,1] [2,2,1] [4,1,1] are legal. The 8-bit warp-tile check had no gfx1250 case, so it fell through to the gfx9 MFMA expectations. gfx1250 is RDNA-style WMMA: the only 8-bit fragments are 16x16x64 and 16x16x128, and there is no 32x32 WMMA at all, so a 32x32xK tile compiles and returns garbage -- 2,908 of 2,908 wrong-result rows in that same sweep were 32x32x32. The reverse was also true: 16x16x128, which is the tile this bridge actually ships, was being rejected. Measured on the real predicate, gfx1250 fp8 rowcol/tensor quant: 32x32x32 ACCEPT -> REJECT, 16x16x128 REJECT -> ACCEPT, 16x16x64 unchanged. gfx942 and gfx950 are bit-for-bit unchanged across the same probe, as is the per-arch entry count of every other WARP_SUPPORTED_COMBINATIONS row. Content is identical to the same fix on the stacked gate branch, so that branch rebases without a semantic conflict. --- .../ops/gemm/gemm_validation_utils.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py index ad262a249532..2d076d84eecb 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py +++ b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py @@ -45,7 +45,11 @@ def get_warp_size_for_gpu(gpu_target: str) -> int: WARP_SUPPORTED_COMBINATIONS = { - "gfx1250": [[2, 4, 1], [1, 8, 1], [8, 1, 1], [4, 2, 1], [2, 1, 1], [1, 2, 2], [4, 1, 1], [1, 4, 1], [2, 2, 1]], + # gfx1250 is wave32: a block of >4 warps has no launchable kernel entry (every + # 8-warp map -- [2,4,1] [4,2,1] [1,8,1] [8,1,1] -- aborts at launch with + # "cannot find symbol", 3,220/3,220 rows in a GPU sweep), and [1,2,2] (warp_k=2) + # compiles and then returns wrong results (GPU-measured max_rel 1.37). + "gfx1250": [[1, 4, 1], [2, 1, 1], [2, 2, 1], [4, 1, 1]], "gfx90a": [ [1, 4, 1], [2, 2, 1], @@ -1411,6 +1415,23 @@ def _validate_fp8_mfma_warp_tile_k( # gfx950 doubles the K-block: # MFMA_F32_16x16x256_F8 (warp_tile_m=16) → warp_tile_k=128 # MFMA_F32_32x32x128_F8 (warp_tile_m=32) → warp_tile_k=64 + if gpu_target == "gfx1250": + # gfx1250 is wave32 RDNA-style WMMA, not MFMA. The only 8-bit fragments + # are V_WMMA_*_16x16x64 and 16x16x128; there is no 32x32 WMMA, so a + # 32x32xK warp tile compiles and then returns garbage (GPU-measured: + # 2,908/2,908 wrong-result rows were 32x32x32). Both K depths are + # GPU-validated for RowColQuant / TensorQuant. + if warp_tile_m != 16: + return False, ( + f"On {gpu_target} the only 8-bit WMMA warp tile is 16x16xK, " + f"got warp_tile_m={warp_tile_m}{suffix}" + ) + if warp_tile_k not in (64, 128): + return False, ( + f"For {a_datatype} on {gpu_target}, warp_tile_m=16 requires " + f"warp_tile_k in (64, 128), got warp_tile_k={warp_tile_k}{suffix}" + ) + return True, "" if gpu_target == "gfx950": expected_k = 64 if warp_tile_m == 32 else 128 else: From 99c7b1e1830c696d90b6dc970feaf5ec0ffa40a1 Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk <3836908+ozturkosu@users.noreply.github.com> Date: Wed, 9 Sep 2026 21:14:19 -0400 Subject: [PATCH 06/24] fix(ck-tile): refuse to build a gfx1250 grouped-quant library with an invalid tile Listing gfx1250 in kSupportedArchs makes the runtime arch check accept the target, but only the Python helper picks an arch-correct tile. The CMake/codegen path does not: unified_grouped_gemm_{rowcolquant,tensorquant}_codegen.py::_default_config() takes no arch argument and hard-codes the gfx9 tile. A CMake-built gfx1250 library would therefore initialize cleanly and then execute a fragment that does not exist on the device, which compiles and returns garbage rather than failing. gfx1250 is not reachable through CMake today because DESIRED_TARGETS omits it, but that is an accident of omission rather than a decision, and it stops protecting the moment someone adds the target. Make the bridge itself fail closed instead. The kernel's tile is already a compile-time property of the generated header, so this is a static_assert rather than a runtime check: an invalid gfx1250 library cannot be produced at all. The rules mirror the Python validator -- on gfx1250 the only 8-bit WMMA fragments are 16x16x64 and 16x16x128, there is no 32x32 WMMA, and a block of more than four warps has no launchable kernel entry on wave32. The test is exact-gfx1250, not gfx12-family: gfx1200/gfx1201 have a 16x16x16 8-bit fragment and are not in kSupportedArchs. Prefix matching accepts the suffixed form ("gfx1250:xnack-") that hipDeviceProp_t reports. Verified against a standalone mock of the generated header, 11 cases: legal gfx1250 tiles and both warp-map shapes compile; 32x32x32, 32x32x16, 16x16x32 and the 8-warp maps are refused; gfx942, gfx950 and an undefined GFX_ARCH are entirely unaffected. --- .../grouped_gemm_rowcolquant_ctypes_lib.cpp | 39 +++++++++++++++++++ .../grouped_gemm_tensorquant_ctypes_lib.cpp | 39 +++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp index 85c2bde948f6..1b9fbc281fa6 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp +++ b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp @@ -93,6 +93,45 @@ static bool is_supported_arch(const std::string& arch) // actually observed, so gfx942/gfx950 callers are not restricted by it. static bool is_gfx12_arch(const std::string& arch) { return arch.rfind("gfx12", 0) == 0; } +// --------------------------------------------------------------------------- +// Compile-time gfx1250 tile guard. +// +// The runtime helper in grouped_gemm_*_utils.py picks an arch-correct tile, but the +// CMake/codegen path does not: unified_grouped_gemm_*_codegen.py::_default_config() +// takes no arch and hard-codes the gfx9 tile. Listing gfx1250 in kSupportedArchs above +// therefore means a CMake-built gfx1250 library would initialize cleanly and then run a +// tile that does not exist on the device -- which compiles and returns garbage rather +// than failing. +// +// Rather than trust the build to pick correctly, refuse to produce such a library at +// all. gfx1250 is wave32 with RDNA-style WMMA: the only 8-bit fragments are 16x16x64 +// and 16x16x128 (there is no 32x32 WMMA), and a block of more than four warps has no +// launchable kernel entry. These are the same rules the Python validator enforces. +// +// The test is exact-gfx1250, not gfx12-family: gfx1200/gfx1201 have a different 8-bit +// fragment (16x16x16) and are not in kSupportedArchs anyway. +static constexpr bool ct_starts_with(const char* s, const char* prefix) +{ + return *prefix == '\0' ? true : (*s == *prefix && ct_starts_with(s + 1, prefix + 1)); +} + +static constexpr bool kCompiledForGfx1250 = ct_starts_with(GFX_ARCH, "gfx1250"); + +static_assert(!kCompiledForGfx1250 || SelectedKernel::WarpTileM == 16, + "gfx1250 has no 32x32 WMMA fragment: warp_tile_m must be 16. This kernel " + "would compile and then return wrong results on the device."); + +static_assert(!kCompiledForGfx1250 || SelectedKernel::WarpTileK == 64 || + SelectedKernel::WarpTileK == 128, + "gfx1250 8-bit WMMA fragments are 16x16x64 and 16x16x128 only: " + "warp_tile_k must be 64 or 128."); + +static_assert(!kCompiledForGfx1250 || + (SelectedKernel::WarpPerBlock_M * SelectedKernel::WarpPerBlock_N * + SelectedKernel::WarpPerBlock_K) <= 4, + "gfx1250 is wave32: a block of more than four warps has no launchable " + "kernel entry and aborts at launch with a missing device symbol."); + extern "C" { /** diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp index cc4748d68bd3..9e76d5c53f53 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp +++ b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp @@ -91,6 +91,45 @@ static bool is_supported_arch(const std::string& arch) return false; } +// --------------------------------------------------------------------------- +// Compile-time gfx1250 tile guard. +// +// The runtime helper in grouped_gemm_*_utils.py picks an arch-correct tile, but the +// CMake/codegen path does not: unified_grouped_gemm_*_codegen.py::_default_config() +// takes no arch and hard-codes the gfx9 tile. Listing gfx1250 in kSupportedArchs above +// therefore means a CMake-built gfx1250 library would initialize cleanly and then run a +// tile that does not exist on the device -- which compiles and returns garbage rather +// than failing. +// +// Rather than trust the build to pick correctly, refuse to produce such a library at +// all. gfx1250 is wave32 with RDNA-style WMMA: the only 8-bit fragments are 16x16x64 +// and 16x16x128 (there is no 32x32 WMMA), and a block of more than four warps has no +// launchable kernel entry. These are the same rules the Python validator enforces. +// +// The test is exact-gfx1250, not gfx12-family: gfx1200/gfx1201 have a different 8-bit +// fragment (16x16x16) and are not in kSupportedArchs anyway. +static constexpr bool ct_starts_with(const char* s, const char* prefix) +{ + return *prefix == '\0' ? true : (*s == *prefix && ct_starts_with(s + 1, prefix + 1)); +} + +static constexpr bool kCompiledForGfx1250 = ct_starts_with(GFX_ARCH, "gfx1250"); + +static_assert(!kCompiledForGfx1250 || SelectedKernel::WarpTileM == 16, + "gfx1250 has no 32x32 WMMA fragment: warp_tile_m must be 16. This kernel " + "would compile and then return wrong results on the device."); + +static_assert(!kCompiledForGfx1250 || SelectedKernel::WarpTileK == 64 || + SelectedKernel::WarpTileK == 128, + "gfx1250 8-bit WMMA fragments are 16x16x64 and 16x16x128 only: " + "warp_tile_k must be 64 or 128."); + +static_assert(!kCompiledForGfx1250 || + (SelectedKernel::WarpPerBlock_M * SelectedKernel::WarpPerBlock_N * + SelectedKernel::WarpPerBlock_K) <= 4, + "gfx1250 is wave32: a block of more than four warps has no launchable " + "kernel entry and aborts at launch with a missing device symbol."); + extern "C" { /** From fd4a1b3c22de98d7581faad3393d0c0cfaf4f2a7 Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk <3836908+ozturkosu@users.noreply.github.com> Date: Wed, 9 Sep 2026 21:17:10 -0400 Subject: [PATCH 07/24] docs(ck-tile): correct the 8-warp gfx1250 claim to match the measured data The comment on the gfx1250 warp-map list, and the static_assert message that echoed it, both said an 8-warp block "has no launchable kernel entry" and cited 3,220/3,220 aborted rows. Recounting the 14,208-row sweep archive that the claim rests on, that is not what the data shows: LAUNCH-ABORT 3352 CLEAN-REJECT 664 FAIL 112 <- wrong answers, not aborts PASS 192 <- correct, all 16x16x64, 192 distinct kernels So 8-warp maps are not uniformly broken. They are nondeterministic: re-running the same surface flips hundreds of rows between abort, pass and fail. That is a stronger reason to reject them, not a weaker one -- a configuration that is sometimes correct can survive a clean validation run and reach users, which a configuration that always aborts cannot. Rejecting them therefore stands; only the stated justification changes. The numbers cited previously also came from a different, smaller sweep than the archive they were attributed to. This diverges from the identical comment on the stacked gate branch, which carries the same overstatement and should receive the same correction. --- .../ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp | 6 ++++-- .../ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp | 6 ++++-- .../tile_engine/ops/gemm/gemm_validation_utils.py | 12 ++++++++---- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp index 1b9fbc281fa6..791c88624c04 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp +++ b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp @@ -129,8 +129,10 @@ static_assert(!kCompiledForGfx1250 || SelectedKernel::WarpTileK == 64 || static_assert(!kCompiledForGfx1250 || (SelectedKernel::WarpPerBlock_M * SelectedKernel::WarpPerBlock_N * SelectedKernel::WarpPerBlock_K) <= 4, - "gfx1250 is wave32: a block of more than four warps has no launchable " - "kernel entry and aborts at launch with a missing device symbol."); + "gfx1250 is wave32: a block of more than four warps is unreliable there " + "-- in a 14,208-row sweep the 8-warp maps aborted at launch 3,352 times " + "and returned wrong results 112 times, while also passing 192 times. " + "Nondeterministic, so refused rather than trusted."); extern "C" { diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp index 9e76d5c53f53..ce976a769a14 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp +++ b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp @@ -127,8 +127,10 @@ static_assert(!kCompiledForGfx1250 || SelectedKernel::WarpTileK == 64 || static_assert(!kCompiledForGfx1250 || (SelectedKernel::WarpPerBlock_M * SelectedKernel::WarpPerBlock_N * SelectedKernel::WarpPerBlock_K) <= 4, - "gfx1250 is wave32: a block of more than four warps has no launchable " - "kernel entry and aborts at launch with a missing device symbol."); + "gfx1250 is wave32: a block of more than four warps is unreliable there " + "-- in a 14,208-row sweep the 8-warp maps aborted at launch 3,352 times " + "and returned wrong results 112 times, while also passing 192 times. " + "Nondeterministic, so refused rather than trusted."); extern "C" { diff --git a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py index 2d076d84eecb..df0511266798 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py +++ b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py @@ -45,10 +45,14 @@ def get_warp_size_for_gpu(gpu_target: str) -> int: WARP_SUPPORTED_COMBINATIONS = { - # gfx1250 is wave32: a block of >4 warps has no launchable kernel entry (every - # 8-warp map -- [2,4,1] [4,2,1] [1,8,1] [8,1,1] -- aborts at launch with - # "cannot find symbol", 3,220/3,220 rows in a GPU sweep), and [1,2,2] (warp_k=2) - # compiles and then returns wrong results (GPU-measured max_rel 1.37). + # gfx1250 is wave32, and a block of more than four warps is unreliable there. + # Over a 14,208-row GPU sweep the 8-warp maps ([2,4,1] [4,2,1] [1,8,1] [8,1,1]) + # produced 3,352 launch aborts ("cannot find symbol"), 664 clean rejects, 112 + # WRONG RESULTS -- and 192 correct passes. They are not uniformly broken; they + # are nondeterministic, and re-running the same surface flips hundreds of rows + # between abort, pass and fail. A config that is sometimes right is worse than + # one that always fails, because a single clean run can ship it. + # [1,2,2] (warp_k=2) compiles and then returns wrong results (max_rel 1.37). "gfx1250": [[1, 4, 1], [2, 1, 1], [2, 2, 1], [4, 1, 1]], "gfx90a": [ [1, 4, 1], From 13a0f7dfcfaa15f9ea67d3da31e45ad4117dd14a Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk <3836908+ozturkosu@users.noreply.github.com> Date: Wed, 9 Sep 2026 21:24:06 -0400 Subject: [PATCH 08/24] docs(ck-tile): separate the warp-map and warp-tile failure modes The previous correction fixed one overstatement and introduced another: it said the 8-warp maps "returned wrong results 112 times". Cross-tabulating warp map against warp tile on the same 14,208-row archive shows those 112 rows are not attributable to the warp map at all. 8-warp + 16x16x64 (legal tile): 2168 abort, 424 reject, 192 pass, 0 FAIL 8-warp + 32x32x32 (illegal tile): 1184 abort, 240 reject, 112 FAIL <=4-warp + 32x32x32: 1206 reject, 3114 FAIL All 3,226 wrong-result rows in the archive use 32x32x32, and no other warp tile produces one. The two variables were confounded: every "8-warp wrong answer" is a 32x32x32 wrong answer that happens to also be 8-warp. So the two rejections have different justifications, and the comments now say so: 32x32x32 is the sole source of silent wrong answers -- highest severity. 8-warp maps are unreliable, not incorrect: mostly abort, sometimes pass, and the split moves between runs. Still rejected, but for dependability. Both remain rejected; only the reasoning is now attributable to the data. --- .../grouped_gemm_rowcolquant_ctypes_lib.cpp | 8 +++---- .../grouped_gemm_tensorquant_ctypes_lib.cpp | 8 +++---- .../ops/gemm/gemm_validation_utils.py | 24 ++++++++++--------- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp index 791c88624c04..0797bcbe990f 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp +++ b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp @@ -129,10 +129,10 @@ static_assert(!kCompiledForGfx1250 || SelectedKernel::WarpTileK == 64 || static_assert(!kCompiledForGfx1250 || (SelectedKernel::WarpPerBlock_M * SelectedKernel::WarpPerBlock_N * SelectedKernel::WarpPerBlock_K) <= 4, - "gfx1250 is wave32: a block of more than four warps is unreliable there " - "-- in a 14,208-row sweep the 8-warp maps aborted at launch 3,352 times " - "and returned wrong results 112 times, while also passing 192 times. " - "Nondeterministic, so refused rather than trusted."); + "gfx1250 is wave32: a block of more than four warps is not dependable " + "there. Paired with a legal tile in a 14,208-row sweep the 8-warp maps " + "aborted at launch 2,168 times and passed 192 times, with no wrong " + "answers -- unreliable rather than incorrect, so refused here."); extern "C" { diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp index ce976a769a14..b33606256767 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp +++ b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp @@ -127,10 +127,10 @@ static_assert(!kCompiledForGfx1250 || SelectedKernel::WarpTileK == 64 || static_assert(!kCompiledForGfx1250 || (SelectedKernel::WarpPerBlock_M * SelectedKernel::WarpPerBlock_N * SelectedKernel::WarpPerBlock_K) <= 4, - "gfx1250 is wave32: a block of more than four warps is unreliable there " - "-- in a 14,208-row sweep the 8-warp maps aborted at launch 3,352 times " - "and returned wrong results 112 times, while also passing 192 times. " - "Nondeterministic, so refused rather than trusted."); + "gfx1250 is wave32: a block of more than four warps is not dependable " + "there. Paired with a legal tile in a 14,208-row sweep the 8-warp maps " + "aborted at launch 2,168 times and passed 192 times, with no wrong " + "answers -- unreliable rather than incorrect, so refused here."); extern "C" { diff --git a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py index df0511266798..0bb284ebe994 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py +++ b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py @@ -45,14 +45,14 @@ def get_warp_size_for_gpu(gpu_target: str) -> int: WARP_SUPPORTED_COMBINATIONS = { - # gfx1250 is wave32, and a block of more than four warps is unreliable there. - # Over a 14,208-row GPU sweep the 8-warp maps ([2,4,1] [4,2,1] [1,8,1] [8,1,1]) - # produced 3,352 launch aborts ("cannot find symbol"), 664 clean rejects, 112 - # WRONG RESULTS -- and 192 correct passes. They are not uniformly broken; they - # are nondeterministic, and re-running the same surface flips hundreds of rows - # between abort, pass and fail. A config that is sometimes right is worse than - # one that always fails, because a single clean run can ship it. - # [1,2,2] (warp_k=2) compiles and then returns wrong results (max_rel 1.37). + # gfx1250 is wave32, and a block of more than four warps is not dependable there. + # Measured over a 14,208-row GPU sweep, the 8-warp maps ([2,4,1] [4,2,1] [1,8,1] + # [8,1,1]) paired with a legal 16x16x64 tile gave 2,168 launch aborts ("cannot + # find symbol"), 424 clean rejects, 192 correct passes -- and ZERO wrong answers. + # They do not corrupt results on their own; they are simply unreliable, and + # re-running the same surface flips hundreds of rows between abort and pass. + # [1,2,2] (warp_k=2) is a different failure: it compiles and returns wrong + # results (max_rel 1.37). "gfx1250": [[1, 4, 1], [2, 1, 1], [2, 2, 1], [4, 1, 1]], "gfx90a": [ [1, 4, 1], @@ -1422,9 +1422,11 @@ def _validate_fp8_mfma_warp_tile_k( if gpu_target == "gfx1250": # gfx1250 is wave32 RDNA-style WMMA, not MFMA. The only 8-bit fragments # are V_WMMA_*_16x16x64 and 16x16x128; there is no 32x32 WMMA, so a - # 32x32xK warp tile compiles and then returns garbage (GPU-measured: - # 2,908/2,908 wrong-result rows were 32x32x32). Both K depths are - # GPU-validated for RowColQuant / TensorQuant. + # 32x32xK warp tile compiles and then returns garbage. This is the sole + # source of silent wrong answers on this surface: all 3,226 wrong-result + # rows of a 14,208-row sweep used 32x32x32, and no other warp tile + # produced one. Both legal K depths are GPU-validated for + # RowColQuant / TensorQuant. if warp_tile_m != 16: return False, ( f"On {gpu_target} the only 8-bit WMMA warp tile is 16x16xK, " From ce25d60ffb4e7833b2497b411f16223160802f25 Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk <3836908+ozturkosu@users.noreply.github.com> Date: Wed, 9 Sep 2026 21:36:16 -0400 Subject: [PATCH 09/24] fix(ck-tile): close two holes in the gfx1250 guards found in review Both guards added earlier advertised protection they did not fully provide. The static_assert capped warps-per-block at four, but [1,2,2] has a product of four and passed it -- and [1,2,2] is the one warp map this PR documents as compiling and then returning wrong results (max_rel 1.37). The corrupting case was the one getting through. warp_k is now required to be 1, as its own rule rather than a side effect of the cap. WarpTileN was never asserted at all, so 16x32xK also passed; it is now pinned to 16. The validator guards were bypassed by a suffixed architecture name. Both WARP_SUPPORTED_COMBINATIONS.get() and the gfx1250 warp-tile branch compared the raw gpu_target, and an unrecognized key falls through permissively -- so a device reporting "gfx1250:xnack-" disabled the entire new restriction and landed in the branch that accepts 32x32x32, the sole measured source of wrong answers. The file already computed a normalized base_gpu_target twenty lines away; both sites now normalize the same way. Verified against the standalone mock, 9 cases: [1,2,2] and 16x32x128 are now refused, every previously accepted gfx1250 tile still compiles, suffixed gfx1250 still resolves, and gfx942/gfx950 remain unaffected including for [1,2,2]. --- .../ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp | 11 +++++++++++ .../ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp | 11 +++++++++++ .../tile_engine/ops/gemm/gemm_validation_utils.py | 9 +++++++-- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp index 0797bcbe990f..d29ac674063f 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp +++ b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp @@ -126,6 +126,17 @@ static_assert(!kCompiledForGfx1250 || SelectedKernel::WarpTileK == 64 || "gfx1250 8-bit WMMA fragments are 16x16x64 and 16x16x128 only: " "warp_tile_k must be 64 or 128."); +static_assert(!kCompiledForGfx1250 || SelectedKernel::WarpTileN == 16, + "gfx1250 8-bit WMMA fragments are 16x16xK: warp_tile_n must be 16."); + +// warp_k > 1 is rejected separately from the warps-per-block cap below. The [1,2,2] +// map has a product of 4 and would pass that cap on its own, but it is measured to +// compile and then return wrong results (max_rel 1.37). It is the corrupting case, +// not merely an undependable one, so it gets its own rule. +static_assert(!kCompiledForGfx1250 || SelectedKernel::WarpPerBlock_K == 1, + "gfx1250: warp_k must be 1. A warp_k of 2 (the [1,2,2] map) compiles " + "and then returns wrong results."); + static_assert(!kCompiledForGfx1250 || (SelectedKernel::WarpPerBlock_M * SelectedKernel::WarpPerBlock_N * SelectedKernel::WarpPerBlock_K) <= 4, diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp index b33606256767..f0b7a6b52ed6 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp +++ b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp @@ -124,6 +124,17 @@ static_assert(!kCompiledForGfx1250 || SelectedKernel::WarpTileK == 64 || "gfx1250 8-bit WMMA fragments are 16x16x64 and 16x16x128 only: " "warp_tile_k must be 64 or 128."); +static_assert(!kCompiledForGfx1250 || SelectedKernel::WarpTileN == 16, + "gfx1250 8-bit WMMA fragments are 16x16xK: warp_tile_n must be 16."); + +// warp_k > 1 is rejected separately from the warps-per-block cap below. The [1,2,2] +// map has a product of 4 and would pass that cap on its own, but it is measured to +// compile and then return wrong results (max_rel 1.37). It is the corrupting case, +// not merely an undependable one, so it gets its own rule. +static_assert(!kCompiledForGfx1250 || SelectedKernel::WarpPerBlock_K == 1, + "gfx1250: warp_k must be 1. A warp_k of 2 (the [1,2,2] map) compiles " + "and then returns wrong results."); + static_assert(!kCompiledForGfx1250 || (SelectedKernel::WarpPerBlock_M * SelectedKernel::WarpPerBlock_N * SelectedKernel::WarpPerBlock_K) <= 4, diff --git a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py index 0bb284ebe994..ea1e4c45c91e 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py +++ b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py @@ -328,7 +328,12 @@ def validate_warp_configuration( current_combination = [warp_m, warp_n, warp_k] - allowed_combinations = WARP_SUPPORTED_COMBINATIONS.get(gpu_name, {}) + # Normalize first: a device reports "gfx1250:xnack-", and an unrecognized key + # falls through permissively below, which would disable this restriction on the + # very target it exists for. + allowed_combinations = WARP_SUPPORTED_COMBINATIONS.get( + gpu_name.split(":")[0] if gpu_name else gpu_name, {} + ) if not allowed_combinations: # If GPU not recognized, try to be permissive but log warning logging.warning(f"No warp_[m/n/k] combinations found for GPU: {gpu_name}") @@ -1419,7 +1424,7 @@ def _validate_fp8_mfma_warp_tile_k( # gfx950 doubles the K-block: # MFMA_F32_16x16x256_F8 (warp_tile_m=16) → warp_tile_k=128 # MFMA_F32_32x32x128_F8 (warp_tile_m=32) → warp_tile_k=64 - if gpu_target == "gfx1250": + if (gpu_target.split(":")[0] if gpu_target else gpu_target) == "gfx1250": # gfx1250 is wave32 RDNA-style WMMA, not MFMA. The only 8-bit fragments # are V_WMMA_*_16x16x64 and 16x16x128; there is no 32x32 WMMA, so a # 32x32xK warp tile compiles and then returns garbage. This is the sole From 678fb564b4548a8bbadcba38130f94fb38eb560f Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk <3836908+ozturkosu@users.noreply.github.com> Date: Thu, 10 Sep 2026 01:15:38 -0400 Subject: [PATCH 10/24] fix(ck-tile): scope the gfx1250 warp-map restriction to this operator Narrowing WARP_SUPPORTED_COMBINATIONS["gfx1250"] from nine warp maps to four was a regression. That table is shared by seven instance builders, and one of the maps removed, 4x2x1, is plain GEMM's default gfx1250 WMMA configuration -- issue #11161 reported it returning wrong results, PR #11175 fixed it, and the fix was verified correct on device. Measured on the predicate: validate_warp_configuration(4, 2, 1, "gfx1250") develop ACCEPT after my change REJECT <- deleted another operator's working default The table is now byte-identical to develop again. The restriction itself is still needed, but only here: these two grouped quant bridges do not emit a loadable kernel entry for >4-warp blocks on gfx1250, so those kernels abort at launch with "cannot find symbol" -- 3,220 rows in a GPU sweep. That is a property of this bridge's code generation, not of the hardware, which is precisely why it does not belong in a hardware capability table. It now lives in validate_gemm_rowcol_tensor_quant(), alongside the warp-tile rule that was already scoped to the four quant validators. warp_k > 1 keeps its own rule rather than riding on the warps-per-block cap: [1,2,2] has a product of 4 and would otherwise pass. Verified: plain GEMM's 4x2x1 default is accepted again; this operator still rejects 4x2x1, 2x4x1, 4x4x1 and 1x2x2; 32x32x32 is still rejected; the legal 16x16x128 tile is still accepted; and across gfx90a, gfx942 and gfx950 there are zero decision changes. --- .../ops/gemm/gemm_validation_utils.py | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py index ea1e4c45c91e..d1a5282d5d6d 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py +++ b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py @@ -45,15 +45,7 @@ def get_warp_size_for_gpu(gpu_target: str) -> int: WARP_SUPPORTED_COMBINATIONS = { - # gfx1250 is wave32, and a block of more than four warps is not dependable there. - # Measured over a 14,208-row GPU sweep, the 8-warp maps ([2,4,1] [4,2,1] [1,8,1] - # [8,1,1]) paired with a legal 16x16x64 tile gave 2,168 launch aborts ("cannot - # find symbol"), 424 clean rejects, 192 correct passes -- and ZERO wrong answers. - # They do not corrupt results on their own; they are simply unreliable, and - # re-running the same surface flips hundreds of rows between abort and pass. - # [1,2,2] (warp_k=2) is a different failure: it compiles and returns wrong - # results (max_rel 1.37). - "gfx1250": [[1, 4, 1], [2, 1, 1], [2, 2, 1], [4, 1, 1]], + "gfx1250": [[2, 4, 1], [1, 8, 1], [8, 1, 1], [4, 2, 1], [2, 1, 1], [1, 2, 2], [4, 1, 1], [1, 4, 1], [2, 2, 1]], "gfx90a": [ [1, 4, 1], [2, 2, 1], @@ -1474,6 +1466,28 @@ def validate_gemm_rowcol_tensor_quant( gpu_target: str, ) -> Tuple[bool, str]: """Validate RowColQuant / TensorQuant GEMM-specific constraints.""" + # gfx1250 warp-map restriction, deliberately scoped to THIS operator rather than + # placed in WARP_SUPPORTED_COMBINATIONS. That table is shared by seven instance + # builders, and plain GEMM's default gfx1250 WMMA config is an 8-warp 4x2x1 map + # that issue #11161 / PR #11175 fixed and verified correct on device -- removing + # it from the shared table would delete another operator's working default. + # + # These two grouped quant bridges are different: their codegen does not emit a + # loadable entry for >4-warp blocks on gfx1250, so those kernels abort at launch + # with "cannot find symbol" (3,220 rows in a GPU sweep). That is a property of + # this bridge's code generation, not of the hardware, so the rule belongs here. + if (gpu_target.split(":")[0] if gpu_target else gpu_target) == "gfx1250": + if warp_m * warp_n * warp_k > 4: + return False, ( + f"On gfx1250 these grouped quant bridges do not emit a loadable " + f"kernel entry for blocks of more than four warps; got " + f"{warp_m}x{warp_n}x{warp_k} ({warp_m * warp_n * warp_k} warps)" + ) + if warp_k > 1: + return False, ( + f"On gfx1250 warp_k must be 1 for these bridges; got warp_k={warp_k}" + ) + whole_workgroup_cover_valid, whole_workgroup_cover_error = ( validate_whole_wg_cover_configuration( tile_m, From 6795101c1380ff4bd53a6542f50b69ca4e81d9c0 Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk <3836908+ozturkosu@users.noreply.github.com> Date: Thu, 10 Sep 2026 02:03:26 -0400 Subject: [PATCH 11/24] fix(ck-tile): align the 8-warp figures to one archive, and fold the gfx12 test Two findings from an independent review of an external code review. The Python comment justifying the warp-map rule cited 3,220 abort rows while the C++ assertion message beside it cited 2,168 for what reads as the same population. Both numbers are real, but they come from different sweeps: 3,220 is from an 11,840-row run, while the C++ message quotes the 14,208-row archive. Recounting that archive, the 4,320 8-warp rows split as 3,352 aborts overall and 2,168 when paired with an otherwise legal tile. The comment now quotes both figures from the one archive and records the earlier mixup, since this is the third time numbers from these two sweeps have been merged in a comment. Separately, the M%4 guard called is_gfx12_arch(std::string(GFX_ARCH)) -- building a std::string on every invocation to test a value fixed at compile time, in a file that already had a constexpr arch predicate a hundred lines above. Replaced with a constexpr kCompiledForGfx12, so on gfx9 the branch folds away entirely. The family-versus-exact distinction is preserved and now explicit: kCompiledForGfx12 covers gfx1200/gfx1201/gfx1250 and gates the M%4 guard, because that defect is an unpadded AQ view in a shared kernel header rather than a gfx1250 silicon property; kCompiledForGfx1250 stays exact and gates the warp tile assertions, because the WMMA fragment shapes genuinely differ within the family. Verified across seven arch strings including suffixed forms. --- .../ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp | 13 +++++++------ .../tile_engine/ops/gemm/gemm_validation_utils.py | 7 +++++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp index d29ac674063f..0c8510e43376 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp +++ b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp @@ -88,11 +88,6 @@ static bool is_supported_arch(const std::string& arch) return false; } -// True if this .so was compiled for a gfx12-class target (WMMA, wave32). -// Used to scope the M%4 rejection below to the architecture where the defect is -// actually observed, so gfx942/gfx950 callers are not restricted by it. -static bool is_gfx12_arch(const std::string& arch) { return arch.rfind("gfx12", 0) == 0; } - // --------------------------------------------------------------------------- // Compile-time gfx1250 tile guard. // @@ -117,6 +112,12 @@ static constexpr bool ct_starts_with(const char* s, const char* prefix) static constexpr bool kCompiledForGfx1250 = ct_starts_with(GFX_ARCH, "gfx1250"); +// Family-wide companion for the M%4 guard below. That defect is an unpadded AQ view in +// the shared quant kernel header rather than a gfx1250-specific silicon property, so +// every gfx12xx part is subject to it. Compile-time, so on gfx9 the guard folds away +// instead of constructing a std::string per call to test a constant. +static constexpr bool kCompiledForGfx12 = ct_starts_with(GFX_ARCH, "gfx12"); + static_assert(!kCompiledForGfx1250 || SelectedKernel::WarpTileM == 16, "gfx1250 has no 32x32 WMMA fragment: warp_tile_m must be 16. This kernel " "would compile and then return wrong results on the device."); @@ -334,7 +335,7 @@ int dispatcher_run_gemm(const void* A, // converts a silent wrong answer into a clean refusal without touching code paths // that other architectures depend on. Remove this guard once the AQ tail is // padded upstream. - if(is_gfx12_arch(std::string(GFX_ARCH)) && (M % 4) != 0) + if(kCompiledForGfx12 && (M % 4) != 0) { std::cerr << "dispatcher_run_gemm: M must be a multiple of 4 on " << GFX_ARCH << " for RowColQuant; got M=" << M << ". The per-row A-scale (AQ) tile " diff --git a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py index d1a5282d5d6d..4a3714fdb42e 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py +++ b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py @@ -1474,8 +1474,11 @@ def validate_gemm_rowcol_tensor_quant( # # These two grouped quant bridges are different: their codegen does not emit a # loadable entry for >4-warp blocks on gfx1250, so those kernels abort at launch - # with "cannot find symbol" (3,220 rows in a GPU sweep). That is a property of - # this bridge's code generation, not of the hardware, so the rule belongs here. + # with "cannot find symbol" -- 3,352 of the 4,320 8-warp rows in a 14,208-row + # sweep, and 2,168 of those even when paired with an otherwise legal tile. That + # is a property of this bridge's code generation, not of the hardware, so the + # rule belongs here. (Both figures come from the same archive; an earlier draft + # of this comment quoted 3,220, which is from a different, smaller sweep.) if (gpu_target.split(":")[0] if gpu_target else gpu_target) == "gfx1250": if warp_m * warp_n * warp_k > 4: return False, ( From e1f8f28e24c2e820103b9dcbdcdc66f3c6974c4a Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk Date: Thu, 10 Sep 2026 02:46:00 -0400 Subject: [PATCH 12/24] refactor(ck-tile): make the arch-normalization comment match the code `normalize_gfx_arch` in dispatcher/codegen/codegen_common.py claimed to be "the single source of truth for that rule; do not re-implement it", while gemm_validation_utils.py open-codes `.split(":")[0]` three times. A comment the adjacent code contradicts is worse than no comment: the next reader either believes it and looks for an import that cannot exist, or stops believing the comments in this area at all. Two ways to fix it. Move the helper somewhere both trees can import, or state the constraint honestly. Moving it is not available: the dependency direction is dispatcher -> tile_engine (dispatcher/python/gemm_utils.py imports gemm_validation_utils, not the other way round), so the shared home would have to be inside tile_engine -- the tree that is being deprecated and whose disappearance this bridge work is explicitly preparing for. Parking new shared infrastructure there buys one copy today and a migration later. So: keep two copies, say so, and make the duplication testable rather than incidental. - codegen_common.normalize_gfx_arch: scope the claim to the dispatcher tree, and record why tile_engine cannot import it. - gemm_validation_utils: add a module-private `_base_gfx_arch` and route the pre-existing inline split in validate_lds_capacity through it, so the rule has one definition per tree instead of one per call site. - test_codegen_common.py: assert the two helpers agree on bare, single-suffix, multi-suffix, empty and malformed inputs. Without this the pair is a comment promising something nothing checks, which is the defect being fixed. Tests: dispatcher/tests/test_codegen_common.py 82 -> 86 passed (15 subtests), 0 failed. (cherry picked from commit d490a72058f8c5c355e47c9b3e421756c0e91f94) --- .../dispatcher/codegen/codegen_common.py | 13 +++++- .../dispatcher/tests/test_codegen_common.py | 40 +++++++++++++++++++ .../ops/gemm/gemm_validation_utils.py | 29 +++++++++++++- 3 files changed, 80 insertions(+), 2 deletions(-) diff --git a/projects/composablekernel/dispatcher/codegen/codegen_common.py b/projects/composablekernel/dispatcher/codegen/codegen_common.py index d2581668547a..a99993ddcc72 100644 --- a/projects/composablekernel/dispatcher/codegen/codegen_common.py +++ b/projects/composablekernel/dispatcher/codegen/codegen_common.py @@ -644,7 +644,18 @@ def normalize_gfx_arch(arch: str) -> str: once at the boundary instead of scattering substring tests that happen to tolerate the suffix. - This is the single source of truth for that rule; do not re-implement it. + Single source of truth *for the dispatcher tree*: everything under + ``dispatcher/`` must call this rather than open-coding ``split(":")``. + + It is deliberately not claimed to be repo-wide, because it is not. + ``tile_engine/`` cannot import it: the dependency direction is + dispatcher -> tile_engine (``dispatcher/python/gemm_utils.py`` imports + ``gemm_validation_utils``), and tile_engine is on the deprecation path, so + moving the helper there to collapse the two copies would park new shared + infrastructure in the tree that is going away. + ``tile_engine/ops/gemm/gemm_validation_utils.py`` therefore keeps its own + ``_base_gfx_arch``; the two are pinned to identical behaviour by + ``dispatcher/tests/test_codegen_common.py::TestNormalizeGfxArch``. """ return arch.split(":", 1)[0] diff --git a/projects/composablekernel/dispatcher/tests/test_codegen_common.py b/projects/composablekernel/dispatcher/tests/test_codegen_common.py index 9389174d5abe..42bf62dbac82 100644 --- a/projects/composablekernel/dispatcher/tests/test_codegen_common.py +++ b/projects/composablekernel/dispatcher/tests/test_codegen_common.py @@ -657,6 +657,46 @@ def test_unknown_or_malformed_input_is_passed_through(self): self.assertEqual(normalize_gfx_arch("notagfx:weird"), "notagfx") self.assertEqual(normalize_gfx_arch(":xnack-"), "") + def test_agrees_with_the_tile_engine_copy(self): + """The deliberate duplicate in tile_engine must not drift from this one. + + tile_engine cannot import codegen_common (dependency direction is + dispatcher -> tile_engine, and tile_engine is on the deprecation path), + so the rule exists twice on purpose. This test is what keeps the two + honest, and the docstrings on both sides point at it. Skipped rather than + failed if the tile_engine tree is absent, so the dispatcher tests stay + runnable on their own. + """ + te_path = ( + DISPATCHER_DIR.parent + / "tile_engine" + / "ops" + / "gemm" + / "gemm_validation_utils.py" + ) + if not te_path.exists(): + self.skipTest("tile_engine tree not present") + import importlib.util + + spec = importlib.util.spec_from_file_location( + "_te_gemm_validation_utils_for_test", te_path + ) + te = importlib.util.module_from_spec(spec) + spec.loader.exec_module(te) + + for arch in ( + "", + "gfx942", + "gfx950", + "gfx1250", + "gfx1250:xnack-", + "gfx942:sramecc+:xnack-", + "notagfx:weird", + ":xnack-", + ): + with self.subTest(arch=arch): + self.assertEqual(te._base_gfx_arch(arch), normalize_gfx_arch(arch)) + class TestRowColTensorQuantDefaultTile(unittest.TestCase): """gfx1250 tile selection is EXACT, not a gfx12 family match. diff --git a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py index 4a3714fdb42e..fd7bd23e3f4b 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py +++ b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py @@ -4,6 +4,33 @@ import logging from typing import Tuple, List + +def _base_gfx_arch(gpu_target: str) -> str: + """Strip feature suffixes from a gfx target string. + + ``"gfx942:sramecc+:xnack-"`` -> ``"gfx942"``. Empty input is passed through + unchanged; this is a normalizer, not a validator. + + This is the ONE place in this module that knows the rule. Call it at the + boundary of a function rather than re-deriving it at each comparison site -- + the failure mode it prevents is a normalized test sitting a few lines above a + raw one, so that a suffixed target takes different branches of the same + function. + + Duplication note: ``dispatcher/codegen/codegen_common.py`` carries the + identical helper (``normalize_gfx_arch``) and cannot be imported from here. + The dependency direction is dispatcher -> tile_engine (see + ``dispatcher/python/gemm_utils.py``, which imports this module), and + tile_engine is on the deprecation path, so new shared infrastructure must not + be parked here for dispatcher to consume. The two copies are held together by + ``dispatcher/tests/test_codegen_common.py::TestNormalizeGfxArch``, which + asserts they agree on the same inputs. + """ + if not gpu_target: + return gpu_target + return gpu_target.split(":", 1)[0] + + GEMM_PIPELINES = ["mem", "compv3", "compv4"] GEMM_PRESHUFFLE_PIPELINES = ["preshufflev2"] @@ -392,7 +419,7 @@ def validate_lds_capacity( matrix_b_size = (tile_n * tile_k) * element_size(b_datatype) total_tile_in_lds = matrix_a_size + matrix_b_size - base_gpu_target = gpu_target.split(":")[0] if gpu_target else gpu_target + base_gpu_target = _base_gfx_arch(gpu_target) hw_lds_size = LDS_SIZE_MAP.get(base_gpu_target, DEFAULT_LDS_SIZE) double_buffer = pipeline in ["preshufflev2", "compv4"] max_tile_size = hw_lds_size // 2 if double_buffer else hw_lds_size From f01bd5385bb55285b3382717a3bf2e58afdc2979 Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk Date: Thu, 10 Sep 2026 02:47:11 -0400 Subject: [PATCH 13/24] fix(ck-tile): normalize the arch once per validator, not per comparison Reviewers spotted `if gpu_target == "gfx950":` sitting a few lines below a normalized gfx1250 test inside the same function. That is not a cosmetic inconsistency -- it is a live bug, and it predates this branch. _validate_fp8_mfma_warp_tile_k encodes ISA fragment shapes. gfx950 doubles the fp8 K-block relative to gfx90a/gfx942 (MFMA_F32_16x16x256_F8 and MFMA_F32_32x32x128_F8). A raw `== "gfx950"` means "gfx950:sramecc+:xnack-" misses and falls through to the gfx942 branch, so the validator rejects gfx950's correct warp_tile_k and accepts gfx942's wrong one. Suffixed targets are not hypothetical here: projects/composablekernel/CMakeLists.txt sets GPU_TARGETS to "gfx908:xnack+;gfx90a:xnack+;gfx942:xnack+;gfx950:xnack+" for ASAN builds, and that string reaches --gpu_target unmodified. Fixed by normalizing once at function entry into `base_gpu_target` and using it for every arch test in the body, rather than adding a third normalized comparison beside the raw one. Same treatment for the operator-scoped gfx1250 rule in validate_gemm_rowcol_tensor_quant. `gpu_target` itself is left alone so diagnostics still echo what the caller passed. Measured over {gfx90a, gfx942, gfx950, gfx1201, gfx1250} x {bare, suffixed} x {fp8, bf8, fp16, bf16} x warp_tile_m {16,32,64} x warp_tile_k {16,32,64,128,256}, 24 decisions change versus develop and all 24 are on suffixed names: - gfx950:sramecc+:xnack- : 12 changed. 16x16x128 and 32x32x64 now accepted (correct for gfx950), 16x16x64 and 32x32x32 now rejected (those are the gfx942 shapes). - gfx1250:xnack- : 12 changed, matching bare gfx1250 -- the branch this branch added. - every bare target, and suffixed gfx90a/gfx942/gfx1201 : 0 changed. New test file test_gemm_validation_arch_normalization.py pins it: suffixed must equal bare for every arch in the matrix, gfx950's K-block must be the one accepted under a suffix, gfx1250 must refuse the nonexistent 32x32 WMMA fragment either way, and the error text must still show the caller's string. Registered with CTest as dispatcher_test_gemm_validation_arch_normalization; tile_engine has no CTest registration of its own, and the bridge/Old-TE instance-set parity that depends on this module is a dispatcher property. Tests: new file 4 passed / 308 subtests, 0 failed. (cherry picked from commit cb87d4311e7577eca424cca19e59c16f2468237c) --- .../dispatcher/tests/CMakeLists.txt | 16 +++ ...test_gemm_validation_arch_normalization.py | 127 ++++++++++++++++++ .../ops/gemm/gemm_validation_utils.py | 14 +- 3 files changed, 154 insertions(+), 3 deletions(-) create mode 100644 projects/composablekernel/dispatcher/tests/test_gemm_validation_arch_normalization.py diff --git a/projects/composablekernel/dispatcher/tests/CMakeLists.txt b/projects/composablekernel/dispatcher/tests/CMakeLists.txt index 12c9b61eb008..523cc9590e80 100644 --- a/projects/composablekernel/dispatcher/tests/CMakeLists.txt +++ b/projects/composablekernel/dispatcher/tests/CMakeLists.txt @@ -130,6 +130,22 @@ set_tests_properties(dispatcher_test_codegen_common PROPERTIES ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" ) +# Arch-string handling in tile_engine's gemm_validation_utils, which seven GEMM +# instance builders and the dispatcher's own warp gate all share. Registered on the +# dispatcher side because the bridge/Old-TE instance-set parity that depends on it +# is a dispatcher property; tile_engine has no CTest registration of its own. +add_test( + NAME dispatcher_test_gemm_validation_arch_normalization + COMMAND ${Python3_EXECUTABLE} -m pytest -q ${CMAKE_CURRENT_SOURCE_DIR}/test_gemm_validation_arch_normalization.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) + +set_tests_properties(dispatcher_test_gemm_validation_arch_normalization PROPERTIES + LABELS "dispatcher;python;validation" + TIMEOUT 120 + ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" +) + # Arch-gate predicate used by the RowColQuant/TensorQuant GPU tests. CPU-only: it # exercises the string predicate, not a device. Worth its own registration because # the failure mode it guards is a SKIP -- CI stays green while covering nothing. diff --git a/projects/composablekernel/dispatcher/tests/test_gemm_validation_arch_normalization.py b/projects/composablekernel/dispatcher/tests/test_gemm_validation_arch_normalization.py new file mode 100644 index 000000000000..053076442b8b --- /dev/null +++ b/projects/composablekernel/dispatcher/tests/test_gemm_validation_arch_normalization.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +"""Arch-string handling in tile_engine's gemm_validation_utils. + +This module is shared by seven GEMM instance builders, and the dispatcher +bridges gate on it too (dispatcher/python/gemm_utils.py imports it directly and +mirrors its warp gate to keep bridge and Old-TE instance sets identical). A +change to how it reads an architecture string is therefore never local, and the +targets that actually reach it are not always bare: CK's own CMakeLists +configures GPU_TARGETS as "gfx908:xnack+;gfx90a:xnack+;gfx942:xnack+;gfx950:xnack+" +for ASAN builds, and that string is what gets handed to --gpu_target. + +These tests pin the two decisions taken while enabling gfx1250: + + 1. Fragment-shape tests ARE normalized. Which MFMA/WMMA fragment the silicon + has does not depend on xnack or sramecc. + 2. The WARP_SUPPORTED_COMBINATIONS lookup is NOT normalized. See + TestWarpConfigurationLookupIsUnchangedOnGfx9 for the evidence. + +Run: python3 -m pytest tests/test_gemm_validation_arch_normalization.py -v +""" + +import importlib.util +import itertools +import logging +import sys +import unittest +from pathlib import Path + +SCRIPT_DIR = Path(__file__).parent.resolve() +DISPATCHER_DIR = SCRIPT_DIR.parent +CK_ROOT = DISPATCHER_DIR.parent + +_TE_VALIDATION = CK_ROOT / "tile_engine" / "ops" / "gemm" / "gemm_validation_utils.py" + + +def _load_validation_module(): + spec = importlib.util.spec_from_file_location( + "_te_gemm_validation_utils_arch_test", _TE_VALIDATION + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +if not _TE_VALIDATION.exists(): # pragma: no cover - tile_engine not checked out + gvu = None +else: + gvu = _load_validation_module() + + +_SUFFIXES = { + "gfx90a": "gfx90a:sramecc+:xnack-", + "gfx942": "gfx942:sramecc+:xnack-", + "gfx950": "gfx950:sramecc+:xnack-", + "gfx1201": "gfx1201:xnack-", + "gfx1250": "gfx1250:xnack-", +} + + +@unittest.skipIf(gvu is None, "tile_engine tree not present") +class TestFragmentShapeChecksAreNormalized(unittest.TestCase): + """A feature suffix must not change which MFMA/WMMA fragment we believe in. + + _validate_fp8_mfma_warp_tile_k encodes ISA fragment shapes. gfx950 doubles + the fp8 K-block relative to gfx90a/gfx942 (MFMA_F32_16x16x256_F8 and + MFMA_F32_32x32x128_F8). Before normalization, "gfx950:sramecc+:xnack-" fell + through the `== "gfx950"` test into the gfx942 branch, so on an ASAN-style + configure the validator rejected gfx950's correct warp_tile_k and accepted + gfx942's wrong one. Both directions of that are pinned here. + """ + + def test_suffixed_target_matches_bare_target_for_every_arch(self): + for bare, suffixed in _SUFFIXES.items(): + for dtype, wt_m, wt_k in itertools.product( + ("fp8", "bf8", "fp16", "bf16"), (16, 32, 64), (16, 32, 64, 128, 256) + ): + with self.subTest(arch=bare, dtype=dtype, m=wt_m, k=wt_k): + self.assertEqual( + gvu._validate_fp8_mfma_warp_tile_k( + wt_m, wt_m, wt_k, dtype, bare + )[0], + gvu._validate_fp8_mfma_warp_tile_k( + wt_m, wt_m, wt_k, dtype, suffixed + )[0], + ) + + def test_suffixed_gfx950_uses_the_gfx950_k_block_not_the_gfx942_one(self): + for dtype in ("fp8", "bf8"): + for wt_m, good_k, gfx942_k in ((16, 128, 64), (32, 64, 32)): + with self.subTest(dtype=dtype, warp_tile_m=wt_m): + ok, _ = gvu._validate_fp8_mfma_warp_tile_k( + wt_m, wt_m, good_k, dtype, _SUFFIXES["gfx950"] + ) + self.assertTrue(ok, "gfx950 K-block must be accepted") + bad, _ = gvu._validate_fp8_mfma_warp_tile_k( + wt_m, wt_m, gfx942_k, dtype, _SUFFIXES["gfx950"] + ) + self.assertFalse(bad, "gfx942 K-block must not be accepted") + + def test_error_message_echoes_the_caller_supplied_target(self): + # Normalization is internal; diagnostics should show what was passed in + # so a mis-typed --gpu_target is visible in the message. + _, msg = gvu._validate_fp8_mfma_warp_tile_k( + 32, 32, 999, "fp8", _SUFFIXES["gfx950"] + ) + self.assertIn(_SUFFIXES["gfx950"], msg) + + def test_gfx1250_rejects_the_32x32_warp_tile_bare_and_suffixed(self): + # There is no 32x32 WMMA fragment on gfx1250; a 32x32 warp tile compiles + # and returns garbage, so this must be rejected however the target is + # spelled. + for target in ("gfx1250", _SUFFIXES["gfx1250"]): + for dtype in ("fp8", "bf8"): + with self.subTest(target=target, dtype=dtype): + ok, _ = gvu._validate_fp8_mfma_warp_tile_k( + 32, 32, 32, dtype, target + ) + self.assertFalse(ok) + + +if __name__ == "__main__": + logging.disable(logging.CRITICAL) + sys.exit(unittest.main()) diff --git a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py index fd7bd23e3f4b..1c29d6a884bf 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py +++ b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py @@ -1429,6 +1429,14 @@ def _validate_fp8_mfma_warp_tile_k( - warp_tile_m == warp_tile_n (square MFMA requirement) - warp_tile_k matches the ISA-mandated K-block for the given warp_tile_m and gpu_target """ + # Normalize once, here at the entry. Every arch test in this function is about + # which MFMA/WMMA fragment the silicon has, and a feature suffix does not change + # that: "gfx950:sramecc+:xnack-" is a gfx950 and must take the gfx950 K-block. + # Before this, a suffixed gfx950 fell through to the gfx90a/gfx942 branch and was + # told to use warp_tile_k=32/64 instead of 64/128 -- i.e. the correct tile was + # rejected and the wrong one accepted. `gpu_target` is kept unmodified for the + # error messages, which should echo what the caller passed in. + base_gpu_target = _base_gfx_arch(gpu_target) suffix = f" ({op_label})" if op_label else "" if warp_tile_m != warp_tile_n: return False, ( @@ -1443,7 +1451,7 @@ def _validate_fp8_mfma_warp_tile_k( # gfx950 doubles the K-block: # MFMA_F32_16x16x256_F8 (warp_tile_m=16) → warp_tile_k=128 # MFMA_F32_32x32x128_F8 (warp_tile_m=32) → warp_tile_k=64 - if (gpu_target.split(":")[0] if gpu_target else gpu_target) == "gfx1250": + if base_gpu_target == "gfx1250": # gfx1250 is wave32 RDNA-style WMMA, not MFMA. The only 8-bit fragments # are V_WMMA_*_16x16x64 and 16x16x128; there is no 32x32 WMMA, so a # 32x32xK warp tile compiles and then returns garbage. This is the sole @@ -1462,7 +1470,7 @@ def _validate_fp8_mfma_warp_tile_k( f"warp_tile_k in (64, 128), got warp_tile_k={warp_tile_k}{suffix}" ) return True, "" - if gpu_target == "gfx950": + if base_gpu_target == "gfx950": expected_k = 64 if warp_tile_m == 32 else 128 else: expected_k = 32 if warp_tile_m == 32 else 64 @@ -1506,7 +1514,7 @@ def validate_gemm_rowcol_tensor_quant( # is a property of this bridge's code generation, not of the hardware, so the # rule belongs here. (Both figures come from the same archive; an earlier draft # of this comment quoted 3,220, which is from a different, smaller sweep.) - if (gpu_target.split(":")[0] if gpu_target else gpu_target) == "gfx1250": + if _base_gfx_arch(gpu_target) == "gfx1250": if warp_m * warp_n * warp_k > 4: return False, ( f"On gfx1250 these grouped quant bridges do not emit a loadable " From 43bda4396506e7875288fe4694285174b62e38bb Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk Date: Thu, 10 Sep 2026 02:50:38 -0400 Subject: [PATCH 14/24] feat(ck-tile): give the codegen default config an architecture _default_config() in unified_grouped_gemm_{rowcolquant,tensorquant}_codegen.py took no architecture and returned the gfx9 MFMA tile unconditionally, while bindings/ctypes/CMakeLists.txt derives GFX_ARCH from CMAKE_HIP_ARCHITECTURES[0]. Configure for gfx1250 and the two disagree, the compile-time guard fires, and the target fails to build. Three options were on the table. Leave the hard failure. Skip the target with a status message, the way mx_gemm does twelve targets above in the same file. Or wire the architecture in, using the selector rowcol_tensor_quant_default_tile(arch) that already exists in codegen_common and was simply never called. Wiring it in, and the static_assert stays. Skipping is right for mx_gemm because mx_gemm is a single-architecture operator: there is no correct gfx942 build of it to produce, so declining to build one loses nothing. These two bridges are the opposite case -- gfx1250 is a target they are meant to work on -- so a skip would leave the user with a target that silently does not exist and no indication that it should have. Meanwhile wiring the arch in, on its own, makes the build succeed on the strength of an off-tree sweep, so the assertion stays as the floor: it is now unreachable through the supported path and fires only if something upstream of it goes wrong. - _default_config(gfx_arch="") and generate_kernels(..., gfx_arch=""). Default "" reproduces the previous output byte for byte; gfx_arch is ignored when an explicit --config/--config-json supplies tile_configs, because such a config already carries its own tile. - New --gfx-arch CLI flag on both codegens, honoured by --list-names too. - CMake: strip feature suffixes from the derived GFX_ARCH (ASAN configures put "gfx942:sramecc+:xnack-" in CMAKE_HIP_ARCHITECTURES, and GFX_ARCH is compared against kSupportedArchs and the tile guards, which want the bare target), and tell the user to pass --gfx-arch in the message printed when no generated header is found. Verified: no flag gives 128x128x64_2x2x1_32x32x16 as before; --gfx-arch gfx1250 gives 16x64x256_1x4x1_16x16x128; --gfx-arch gfx1250:xnack- gives the same as bare gfx1250. Also makes the drift guard arch-aware, per the related finding. TestDefaultConfigAlignment compared the runtime and codegen defaults at one architecture, so it could not see a divergence that only exists on another -- which is precisely the divergence this change removes. Both cases are now parametrized over gfx942/gfx950/gfx1250 plus a suffixed spelling of two of them. A parametrized equality test can pass for the wrong reason (if _default_config stopped honouring gfx_arch, both sides would fall back to the gfx9 tile together and every case would still pass), so a companion test asserts the gfx1250 and gfx942 default kernel names are disjoint. Confirmed as a real negative test: pinning the codegen back to the gfx9 tile fails 5 of the 11 cases in that class, on gfx1250 and gfx1250:xnack- only. Tests: test_grouped_gemm_rowcolquant_utils.py + test_grouped_gemm_tensorquant_utils.py 108 -> 128 passed, 0 failed. (cherry picked from commit e8ff28c08093b5cffd8e6ae96cf268ef935733b2) --- .../dispatcher/bindings/ctypes/CMakeLists.txt | 20 +++++++- ...nified_grouped_gemm_rowcolquant_codegen.py | 29 +++++++++--- ...nified_grouped_gemm_tensorquant_codegen.py | 29 +++++++++--- .../test_grouped_gemm_rowcolquant_utils.py | 46 ++++++++++++++----- .../test_grouped_gemm_tensorquant_utils.py | 46 ++++++++++++++----- 5 files changed, 132 insertions(+), 38 deletions(-) diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/CMakeLists.txt b/projects/composablekernel/dispatcher/bindings/ctypes/CMakeLists.txt index ce13a7d8e70c..9e5189e3784b 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/CMakeLists.txt +++ b/projects/composablekernel/dispatcher/bindings/ctypes/CMakeLists.txt @@ -351,6 +351,11 @@ if(ROWCOLQUANT_GEMM_KERNEL_HEADERS) else() set(_rowcolquant_gfx_arch "gfx942") endif() + # Strip feature suffixes ("gfx942:sramecc+:xnack-"). CMAKE_HIP_ARCHITECTURES + # carries them for ASAN configures, and GFX_ARCH is compared against + # kSupportedArchs and the compile-time tile guards in the .cpp, which want the + # bare target. Mirrors normalize_gfx_arch() on the Python side. + string(REGEX REPLACE ":.*$" "" _rowcolquant_gfx_arch "${_rowcolquant_gfx_arch}") set(CK_TILE_ROWCOLQUANT_GFX_ARCH "${_rowcolquant_gfx_arch}" CACHE STRING "GFX arch for the CMake-built RowColQuant ctypes .so (default: first CMAKE_HIP_ARCHITECTURES or gfx942)") endif() @@ -365,7 +370,10 @@ else() "No RowColQuant GEMM kernel header found in ${CMAKE_BINARY_DIR}/generated_kernels/. " "dispatcher_grouped_gemm_rowcolquant_lib will NOT be built. " "Run unified_grouped_gemm_rowcolquant_codegen.py --output-dir " - "${CMAKE_BINARY_DIR}/generated_kernels/ before configuring to enable this target.") + "${CMAKE_BINARY_DIR}/generated_kernels/ --gfx-arch before configuring " + "to enable this target. --gfx-arch matters: the built-in default is the gfx9 " + "MFMA tile, and on gfx1250 the ctypes source static_asserts that tile away " + "rather than let it compile into a library that returns wrong results.") # Do not define the target: the source requires force-included symbols (SelectedKernel, # ADataType, KERNEL_NAME, etc.) that only exist in the generated header. Without it # the build fails at compile time, not load time, breaking any all-target build. @@ -397,6 +405,11 @@ if(TENSORQUANT_GEMM_KERNEL_HEADERS) else() set(_tensorquant_gfx_arch "gfx942") endif() + # Strip feature suffixes ("gfx942:sramecc+:xnack-"). CMAKE_HIP_ARCHITECTURES + # carries them for ASAN configures, and GFX_ARCH is compared against + # kSupportedArchs and the compile-time tile guards in the .cpp, which want the + # bare target. Mirrors normalize_gfx_arch() on the Python side. + string(REGEX REPLACE ":.*$" "" _tensorquant_gfx_arch "${_tensorquant_gfx_arch}") set(CK_TILE_TENSORQUANT_GFX_ARCH "${_tensorquant_gfx_arch}" CACHE STRING "GFX arch for the CMake-built TensorQuant ctypes .so (default: first CMAKE_HIP_ARCHITECTURES or gfx942)") endif() @@ -411,7 +424,10 @@ else() "No TensorQuant GEMM kernel header found in ${CMAKE_BINARY_DIR}/generated_kernels/. " "dispatcher_grouped_gemm_tensorquant_lib will NOT be built. " "Run unified_grouped_gemm_tensorquant_codegen.py --output-dir " - "${CMAKE_BINARY_DIR}/generated_kernels/ before configuring to enable this target.") + "${CMAKE_BINARY_DIR}/generated_kernels/ --gfx-arch before configuring " + "to enable this target. --gfx-arch matters: the built-in default is the gfx9 " + "MFMA tile, and on gfx1250 the ctypes source static_asserts that tile away " + "rather than let it compile into a library that returns wrong results.") # Do not define the target: the source requires force-included symbols (SelectedKernel, # ADataType, KERNEL_NAME, etc.) that only exist in the generated header. Without it # the build fails at compile time, not load time, breaking any all-target build. diff --git a/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_rowcolquant_codegen.py b/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_rowcolquant_codegen.py index e12199429dc4..a8c02b0c4cbe 100644 --- a/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_rowcolquant_codegen.py +++ b/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_rowcolquant_codegen.py @@ -33,12 +33,12 @@ from codegen_common import ( ROWCOL_TENSOR_QUANT_BASE_PIPELINE_MAP, - ROWCOL_TENSOR_QUANT_DEFAULT_TILE, ROWCOL_TENSOR_QUANT_DEFAULT_TRAITS, ROWCOL_TENSOR_QUANT_EPILOGUE_MAP, ROWCOL_TENSOR_QUANT_PIPELINE_MAP, ROWCOL_TENSOR_QUANT_SUPPORTED_LAYOUTS, make_rowcolquant_kernel_name, + rowcol_tensor_quant_default_tile, ) logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") @@ -429,14 +429,20 @@ def generate(self, spec: RowColQuantKernelSpec) -> str: # ============================================================================= -def _default_config() -> dict: +def _default_config(gfx_arch: str = "") -> dict: # Traits and tile come from codegen_common so this default and the runtime # default_{fp8,bf8}_config() in grouped_gemm_rowcolquant_utils.py cannot drift. + # + # gfx_arch selects the tile. It defaults to "" -- the gfx9 MFMA tile -- so an + # invocation with no --gfx-arch generates exactly what it generated before. + # Passing "gfx1250" yields the 16x16x128 WMMA tile instead; the gfx9 tile is + # sized for an MFMA fragment that does not exist on that part, and compiles + # there rather than erroring. return { "dtypes": ["fp8", "bf8"], "layouts": list(ROWCOLQUANT_SUPPORTED_LAYOUTS), **ROWCOL_TENSOR_QUANT_DEFAULT_TRAITS, - "tile_configs": [dict(ROWCOL_TENSOR_QUANT_DEFAULT_TILE)], + "tile_configs": [rowcol_tensor_quant_default_tile(gfx_arch)], } @@ -535,10 +541,15 @@ def generate_kernels( output_dir: Path, config: Optional[dict] = None, parallel: bool = True, + gfx_arch: str = "", ) -> List[Path]: - """Generate all RowColQuant kernel headers into output_dir. Returns list of generated .hpp paths.""" + """Generate all RowColQuant kernel headers into output_dir. Returns list of generated .hpp paths. + + `gfx_arch` only selects the built-in default tile and is ignored when `config` + is supplied -- an explicit config already carries its own tile_configs. + """ output_dir.mkdir(parents=True, exist_ok=True) - cfg = config or _default_config() + cfg = config or _default_config(gfx_arch) specs = _build_specs(cfg) if not specs: @@ -596,6 +607,11 @@ def main() -> int: help="Disable parallel generation") parser.add_argument("--list-names", action="store_true", help="Print kernel names that would be generated and exit") + parser.add_argument("--gfx-arch", type=str, default="", + help="GPU target the generated kernels will be compiled for, " + "e.g. gfx1250. Selects the built-in default tile; ignored " + "when --config/--config-json supplies tile_configs. " + "Defaults to the gfx9 MFMA tile.") args = parser.parse_args() cfg: Optional[dict] = None @@ -610,7 +626,7 @@ def main() -> int: cfg = json.load(f) if args.list_names: - specs = _build_specs(cfg or _default_config()) + specs = _build_specs(cfg or _default_config(args.gfx_arch)) for s in specs: print(s.name) return 0 @@ -619,6 +635,7 @@ def main() -> int: output_dir=args.output_dir, config=cfg, parallel=not args.no_parallel, + gfx_arch=args.gfx_arch, ) return 0 if paths else 1 diff --git a/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_tensorquant_codegen.py b/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_tensorquant_codegen.py index 09591e21e32b..88597d2900c0 100644 --- a/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_tensorquant_codegen.py +++ b/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_tensorquant_codegen.py @@ -33,12 +33,12 @@ from codegen_common import ( ROWCOL_TENSOR_QUANT_BASE_PIPELINE_MAP, - ROWCOL_TENSOR_QUANT_DEFAULT_TILE, ROWCOL_TENSOR_QUANT_DEFAULT_TRAITS, ROWCOL_TENSOR_QUANT_EPILOGUE_MAP, ROWCOL_TENSOR_QUANT_PIPELINE_MAP, ROWCOL_TENSOR_QUANT_SUPPORTED_LAYOUTS, make_tensorquant_kernel_name, + rowcol_tensor_quant_default_tile, ) logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") @@ -429,14 +429,20 @@ def generate(self, spec: TensorQuantKernelSpec) -> str: # ============================================================================= -def _default_config() -> dict: +def _default_config(gfx_arch: str = "") -> dict: # Traits and tile come from codegen_common so this default and the runtime # default_{fp8,bf8}_config() in grouped_gemm_tensorquant_utils.py cannot drift. + # + # gfx_arch selects the tile. It defaults to "" -- the gfx9 MFMA tile -- so an + # invocation with no --gfx-arch generates exactly what it generated before. + # Passing "gfx1250" yields the 16x16x128 WMMA tile instead; the gfx9 tile is + # sized for an MFMA fragment that does not exist on that part, and compiles + # there rather than erroring. return { "dtypes": ["fp8", "bf8"], "layouts": list(TENSORQUANT_SUPPORTED_LAYOUTS), **ROWCOL_TENSOR_QUANT_DEFAULT_TRAITS, - "tile_configs": [dict(ROWCOL_TENSOR_QUANT_DEFAULT_TILE)], + "tile_configs": [rowcol_tensor_quant_default_tile(gfx_arch)], } @@ -535,10 +541,15 @@ def generate_kernels( output_dir: Path, config: Optional[dict] = None, parallel: bool = True, + gfx_arch: str = "", ) -> List[Path]: - """Generate all TensorQuant kernel headers into output_dir. Returns list of generated .hpp paths.""" + """Generate all TensorQuant kernel headers into output_dir. Returns list of generated .hpp paths. + + `gfx_arch` only selects the built-in default tile and is ignored when `config` + is supplied -- an explicit config already carries its own tile_configs. + """ output_dir.mkdir(parents=True, exist_ok=True) - cfg = config or _default_config() + cfg = config or _default_config(gfx_arch) specs = _build_specs(cfg) if not specs: @@ -596,6 +607,11 @@ def main() -> int: help="Disable parallel generation") parser.add_argument("--list-names", action="store_true", help="Print kernel names that would be generated and exit") + parser.add_argument("--gfx-arch", type=str, default="", + help="GPU target the generated kernels will be compiled for, " + "e.g. gfx1250. Selects the built-in default tile; ignored " + "when --config/--config-json supplies tile_configs. " + "Defaults to the gfx9 MFMA tile.") args = parser.parse_args() cfg: Optional[dict] = None @@ -610,7 +626,7 @@ def main() -> int: cfg = json.load(f) if args.list_names: - specs = _build_specs(cfg or _default_config()) + specs = _build_specs(cfg or _default_config(args.gfx_arch)) for s in specs: print(s.name) return 0 @@ -619,6 +635,7 @@ def main() -> int: output_dir=args.output_dir, config=cfg, parallel=not args.no_parallel, + gfx_arch=args.gfx_arch, ) return 0 if paths else 1 diff --git a/projects/composablekernel/dispatcher/tests/test_grouped_gemm_rowcolquant_utils.py b/projects/composablekernel/dispatcher/tests/test_grouped_gemm_rowcolquant_utils.py index f30e6d34aa04..74c6ab3f0f77 100644 --- a/projects/composablekernel/dispatcher/tests/test_grouped_gemm_rowcolquant_utils.py +++ b/projects/composablekernel/dispatcher/tests/test_grouped_gemm_rowcolquant_utils.py @@ -304,24 +304,46 @@ def test_header_launch_takes_vector_of_host_args(self): class TestDefaultConfigAlignment: """Ensure default_fp8_config/default_bf8_config stay in sync with _default_config().""" - def _codegen_default_names(self): + # Both defaults are arch-dependent, so comparing them at a single architecture + # cannot see a drift that only exists on another one -- which is exactly the + # drift gfx1250 enablement introduces. Every arch the bridge supports is + # checked, plus a suffixed spelling on each side. + _ARCHES = ["gfx942", "gfx950", "gfx1250", "gfx1250:xnack-", "gfx942:sramecc+:xnack-"] + + def _codegen_default_names(self, gfx_arch=""): sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "codegen")) from unified_grouped_gemm_rowcolquant_codegen import _default_config, _build_specs - specs = _build_specs(_default_config()) + specs = _build_specs(_default_config(gfx_arch)) return {s.name for s in specs} - def test_default_fp8_config_name_in_codegen_defaults(self): - cfg = default_fp8_config() - assert cfg.name in self._codegen_default_names(), ( - f"default_fp8_config().name '{cfg.name}' is not produced by _default_config() " - f"in the codegen. The two defaults have drifted — update one to match the other." + @pytest.mark.parametrize("gfx_arch", _ARCHES) + def test_default_fp8_config_name_in_codegen_defaults(self, gfx_arch): + cfg = default_fp8_config(gfx_arch) + assert cfg.name in self._codegen_default_names(gfx_arch), ( + f"default_fp8_config('{gfx_arch}').name '{cfg.name}' is not produced by " + f"_default_config('{gfx_arch}') in the codegen. The two defaults have " + f"drifted — update one to match the other." ) - def test_default_bf8_config_name_in_codegen_defaults(self): - cfg = default_bf8_config() - assert cfg.name in self._codegen_default_names(), ( - f"default_bf8_config().name '{cfg.name}' is not produced by _default_config() " - f"in the codegen. The two defaults have drifted — update one to match the other." + @pytest.mark.parametrize("gfx_arch", _ARCHES) + def test_default_bf8_config_name_in_codegen_defaults(self, gfx_arch): + cfg = default_bf8_config(gfx_arch) + assert cfg.name in self._codegen_default_names(gfx_arch), ( + f"default_bf8_config('{gfx_arch}').name '{cfg.name}' is not produced by " + f"_default_config('{gfx_arch}') in the codegen. The two defaults have " + f"drifted — update one to match the other." + ) + + def test_gfx1250_default_actually_differs_from_the_gfx9_default(self): + """Guard against the parametrization passing for the wrong reason. + + If _default_config() ever stops honouring gfx_arch, every case above + still passes -- both sides would just fall back to the gfx9 tile + together. Assert the two architectures really do produce different + kernels, so that regression is visible. + """ + assert self._codegen_default_names("gfx1250").isdisjoint( + self._codegen_default_names("gfx942") ) diff --git a/projects/composablekernel/dispatcher/tests/test_grouped_gemm_tensorquant_utils.py b/projects/composablekernel/dispatcher/tests/test_grouped_gemm_tensorquant_utils.py index c6aa22b4fcfa..289ad64d6dee 100644 --- a/projects/composablekernel/dispatcher/tests/test_grouped_gemm_tensorquant_utils.py +++ b/projects/composablekernel/dispatcher/tests/test_grouped_gemm_tensorquant_utils.py @@ -311,24 +311,46 @@ def test_header_does_not_contain_rowcolquant(self): class TestDefaultConfigAlignment: """Ensure default_fp8_config/default_bf8_config stay in sync with _default_config().""" - def _codegen_default_names(self): + # Both defaults are arch-dependent, so comparing them at a single architecture + # cannot see a drift that only exists on another one -- which is exactly the + # drift gfx1250 enablement introduces. Every arch the bridge supports is + # checked, plus a suffixed spelling on each side. + _ARCHES = ["gfx942", "gfx950", "gfx1250", "gfx1250:xnack-", "gfx942:sramecc+:xnack-"] + + def _codegen_default_names(self, gfx_arch=""): sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "codegen")) from unified_grouped_gemm_tensorquant_codegen import _default_config, _build_specs - specs = _build_specs(_default_config()) + specs = _build_specs(_default_config(gfx_arch)) return {s.name for s in specs} - def test_default_fp8_config_name_in_codegen_defaults(self): - cfg = default_fp8_config() - assert cfg.name in self._codegen_default_names(), ( - f"default_fp8_config().name '{cfg.name}' is not produced by _default_config() " - f"in the codegen. The two defaults have drifted — update one to match the other." + @pytest.mark.parametrize("gfx_arch", _ARCHES) + def test_default_fp8_config_name_in_codegen_defaults(self, gfx_arch): + cfg = default_fp8_config(gfx_arch) + assert cfg.name in self._codegen_default_names(gfx_arch), ( + f"default_fp8_config('{gfx_arch}').name '{cfg.name}' is not produced by " + f"_default_config('{gfx_arch}') in the codegen. The two defaults have " + f"drifted — update one to match the other." ) - def test_default_bf8_config_name_in_codegen_defaults(self): - cfg = default_bf8_config() - assert cfg.name in self._codegen_default_names(), ( - f"default_bf8_config().name '{cfg.name}' is not produced by _default_config() " - f"in the codegen. The two defaults have drifted — update one to match the other." + @pytest.mark.parametrize("gfx_arch", _ARCHES) + def test_default_bf8_config_name_in_codegen_defaults(self, gfx_arch): + cfg = default_bf8_config(gfx_arch) + assert cfg.name in self._codegen_default_names(gfx_arch), ( + f"default_bf8_config('{gfx_arch}').name '{cfg.name}' is not produced by " + f"_default_config('{gfx_arch}') in the codegen. The two defaults have " + f"drifted — update one to match the other." + ) + + def test_gfx1250_default_actually_differs_from_the_gfx9_default(self): + """Guard against the parametrization passing for the wrong reason. + + If _default_config() ever stops honouring gfx_arch, every case above + still passes -- both sides would just fall back to the gfx9 tile + together. Assert the two architectures really do produce different + kernels, so that regression is visible. + """ + assert self._codegen_default_names("gfx1250").isdisjoint( + self._codegen_default_names("gfx942") ) From 25f9d41fe79d9b5bf7f976f0f20c246b1caa6e9d Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk <3836908+ozturkosu@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:41:14 -0400 Subject: [PATCH 15/24] fix(ck-tile): reject an unrecognized --gfx-arch instead of silently using gfx9 --gfx-arch was accepted verbatim. It selects the built-in default tile and nothing else, so it never reaches --offload-arch and hipcc never sees it: a typo does not fail, it falls through to the gfx9 MFMA 32x32x16 tile. Generate with "--gfx-arch gfx1205" and compile the result for gfx1250 and you get a kernel that builds cleanly and returns garbage -- the exact failure this branch exists to close, entered through the front door. Validated at the CLI boundary rather than inside rowcol_tensor_quant_default_tile(). Making the tile selector itself raise was the tempting option, but it would delete a decision the panel already accepted: test_codegen_common.py asserts that gfx1200/gfx1201 fall back to the MFMA tile rather than inheriting gfx1250's, and that test documents why the gfx1250 match is exact and not a gfx12 family match. Raising there would replace a reviewed behaviour as a side effect of adding argument checking. The CLI is also the only entry point where an unknown target is genuinely silent -- every other path eventually hands the value to a compiler, which rejects it -- so that is where the check belongs and where its justification is honest. The accepted set is the same four as gemm_utils._SUPPORTED_ARCHES, pinned by a test, so the bridge does not advertise different support on its codegen and its runner. Not taken from arch_specs.json: that file omits gfx1250 entirely and lists gfx908/gfx1100/gfx1200/gfx1201, for which neither default tile is correct. Suffixed targets are normalized before the check, so the name a real device reports is usable. Behaviour change, deliberate and narrow: "--gfx-arch gfx1200" (and gfx908, gfx1100, gfx1201) previously produced the gfx9 tile and now exits 1. No automated caller passes the flag -- it appears only in a CMake diagnostic aimed at a human -- so this reaches manual invocations only, and only ones that were already getting a tile no one had validated for that target. tests/test_codegen_common.py: 87 -> 93 tests, all passing. CLI verified: gfx1250 and gfx1250:xnack- list kernels and exit 0; gfx1205 and gfx1200 print the supported set and exit 1. (cherry picked from commit 47fdbd53cc634b64487a454680818c1072fcd6bb) --- .../dispatcher/codegen/codegen_common.py | 38 +++++++++++ ...nified_grouped_gemm_rowcolquant_codegen.py | 15 +++- ...nified_grouped_gemm_tensorquant_codegen.py | 15 +++- .../dispatcher/tests/test_codegen_common.py | 68 +++++++++++++++++++ 4 files changed, 132 insertions(+), 4 deletions(-) diff --git a/projects/composablekernel/dispatcher/codegen/codegen_common.py b/projects/composablekernel/dispatcher/codegen/codegen_common.py index a99993ddcc72..090ad01ad4d8 100644 --- a/projects/composablekernel/dispatcher/codegen/codegen_common.py +++ b/projects/composablekernel/dispatcher/codegen/codegen_common.py @@ -1040,6 +1040,44 @@ def rowcol_tensor_quant_default_tile(gfx_arch: str = "") -> dict: return dict(ROWCOL_TENSOR_QUANT_DEFAULT_TILE) +# Architectures the RowColQuant/TensorQuant codegen will accept on its command line. +# Deliberately the same four as dispatcher/python/gemm_utils.py::_SUPPORTED_ARCHES, +# so the bridge does not advertise one supported set on one entry point and a +# different one on another. +# +# Not taken from codegen/arch_specs.json: that file does not list gfx1250 at all, and +# it does list gfx908/gfx1100/gfx1200/gfx1201, none of which these two operators have +# a defensible default tile for -- gfx1200/gfx1201 in particular are WMMA parts whose +# 8-bit fragment is 16x16x16, so both tiles above are wrong for them. +ROWCOL_TENSOR_QUANT_SUPPORTED_ARCHES = ("gfx90a", "gfx942", "gfx950", "gfx1250") + + +def validate_rowcol_tensor_quant_gfx_arch(gfx_arch: str) -> str: + """Normalize and check a caller-supplied gfx target; return the bare target. + + Raises ``ValueError`` for anything outside + ``ROWCOL_TENSOR_QUANT_SUPPORTED_ARCHES``. Empty is allowed and means "not + specified", which selects the gfx9 MFMA tile -- the behaviour every invocation + without the flag had before the flag existed. + + This exists because ``--gfx-arch`` on the two codegen scripts is the one place a + typo is completely silent. Everywhere else a bad target eventually reaches + ``--offload-arch`` and hipcc rejects it; here the value only picks a tile, so + ``--gfx-arch gfx1205`` quietly generates the gfx9 MFMA tile and the result is a + kernel that compiles for gfx1250 and returns garbage -- which is the failure mode + this whole branch exists to close, arriving through the front door. + """ + if not gfx_arch: + return "" + base = normalize_gfx_arch(gfx_arch) + if base not in ROWCOL_TENSOR_QUANT_SUPPORTED_ARCHES: + raise ValueError( + f"Unsupported GPU architecture {gfx_arch!r} (normalized to {base!r}); " + f"supported: {', '.join(ROWCOL_TENSOR_QUANT_SUPPORTED_ARCHES)}." + ) + return base + + # Default traits, shared for the same reason as the tile above. pad_m is enabled # because these kernels are used with M values that are not tile-aligned. ROWCOL_TENSOR_QUANT_DEFAULT_TRAITS = { diff --git a/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_rowcolquant_codegen.py b/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_rowcolquant_codegen.py index a8c02b0c4cbe..61236b0501ee 100644 --- a/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_rowcolquant_codegen.py +++ b/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_rowcolquant_codegen.py @@ -37,6 +37,7 @@ ROWCOL_TENSOR_QUANT_EPILOGUE_MAP, ROWCOL_TENSOR_QUANT_PIPELINE_MAP, ROWCOL_TENSOR_QUANT_SUPPORTED_LAYOUTS, + validate_rowcol_tensor_quant_gfx_arch, make_rowcolquant_kernel_name, rowcol_tensor_quant_default_tile, ) @@ -614,6 +615,16 @@ def main() -> int: "Defaults to the gfx9 MFMA tile.") args = parser.parse_args() + # Validate before anything uses it. This value never reaches a compiler -- it + # only picks a tile -- so an unrecognized target is not caught downstream the way + # a bad --offload-arch would be: it silently falls through to the gfx9 MFMA tile, + # which on WMMA hardware compiles and returns garbage. Refuse instead. + try: + gfx_arch = validate_rowcol_tensor_quant_gfx_arch(args.gfx_arch) + except ValueError as e: + log.error("%s", e) + return 1 + cfg: Optional[dict] = None if args.config_json: try: @@ -626,7 +637,7 @@ def main() -> int: cfg = json.load(f) if args.list_names: - specs = _build_specs(cfg or _default_config(args.gfx_arch)) + specs = _build_specs(cfg or _default_config(gfx_arch)) for s in specs: print(s.name) return 0 @@ -635,7 +646,7 @@ def main() -> int: output_dir=args.output_dir, config=cfg, parallel=not args.no_parallel, - gfx_arch=args.gfx_arch, + gfx_arch=gfx_arch, ) return 0 if paths else 1 diff --git a/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_tensorquant_codegen.py b/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_tensorquant_codegen.py index 88597d2900c0..c32125da35c4 100644 --- a/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_tensorquant_codegen.py +++ b/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_tensorquant_codegen.py @@ -37,6 +37,7 @@ ROWCOL_TENSOR_QUANT_EPILOGUE_MAP, ROWCOL_TENSOR_QUANT_PIPELINE_MAP, ROWCOL_TENSOR_QUANT_SUPPORTED_LAYOUTS, + validate_rowcol_tensor_quant_gfx_arch, make_tensorquant_kernel_name, rowcol_tensor_quant_default_tile, ) @@ -614,6 +615,16 @@ def main() -> int: "Defaults to the gfx9 MFMA tile.") args = parser.parse_args() + # Validate before anything uses it. This value never reaches a compiler -- it + # only picks a tile -- so an unrecognized target is not caught downstream the way + # a bad --offload-arch would be: it silently falls through to the gfx9 MFMA tile, + # which on WMMA hardware compiles and returns garbage. Refuse instead. + try: + gfx_arch = validate_rowcol_tensor_quant_gfx_arch(args.gfx_arch) + except ValueError as e: + log.error("%s", e) + return 1 + cfg: Optional[dict] = None if args.config_json: try: @@ -626,7 +637,7 @@ def main() -> int: cfg = json.load(f) if args.list_names: - specs = _build_specs(cfg or _default_config(args.gfx_arch)) + specs = _build_specs(cfg or _default_config(gfx_arch)) for s in specs: print(s.name) return 0 @@ -635,7 +646,7 @@ def main() -> int: output_dir=args.output_dir, config=cfg, parallel=not args.no_parallel, - gfx_arch=args.gfx_arch, + gfx_arch=gfx_arch, ) return 0 if paths else 1 diff --git a/projects/composablekernel/dispatcher/tests/test_codegen_common.py b/projects/composablekernel/dispatcher/tests/test_codegen_common.py index 42bf62dbac82..b464228ba013 100644 --- a/projects/composablekernel/dispatcher/tests/test_codegen_common.py +++ b/projects/composablekernel/dispatcher/tests/test_codegen_common.py @@ -42,6 +42,8 @@ rowcol_tensor_quant_default_tile, ROWCOL_TENSOR_QUANT_DEFAULT_TILE, ROWCOL_TENSOR_QUANT_DEFAULT_TILE_GFX1250, + ROWCOL_TENSOR_QUANT_SUPPORTED_ARCHES, + validate_rowcol_tensor_quant_gfx_arch, make_aquant_kernel_name, # Shared quant spec-sweep plumbing (see TestQuantSpecSweepHelpers). fp8_warp_tile_k_for_arch, @@ -741,5 +743,71 @@ def test_returns_a_copy_not_the_shared_dict(self): self.assertNotEqual(ROWCOL_TENSOR_QUANT_DEFAULT_TILE_GFX1250["tile_m"], -1) +class TestValidateRowColTensorQuantGfxArch(unittest.TestCase): + """--gfx-arch has to be checked, because nothing downstream checks it. + + The flag only selects a default tile; it never reaches --offload-arch, so an + unrecognized value is not rejected by hipcc the way a bad compile target is. It + just falls through to the gfx9 MFMA tile -- which on gfx1250 compiles and returns + garbage. That is the failure this operator's gfx1250 work exists to prevent, + reachable by a single typo on the command line. + """ + + def test_supported_arches_pass_through_bare(self): + for arch in ROWCOL_TENSOR_QUANT_SUPPORTED_ARCHES: + with self.subTest(arch=arch): + self.assertEqual(validate_rowcol_tensor_quant_gfx_arch(arch), arch) + + def test_feature_suffixes_are_accepted_and_stripped(self): + # A real device reports the suffixed form; refusing it would make the flag + # unusable with the name the hardware gives. + for given, expected in [ + ("gfx1250:xnack-", "gfx1250"), + ("gfx950:sramecc+:xnack-", "gfx950"), + ("gfx942:xnack+", "gfx942"), + ]: + with self.subTest(arch=given): + self.assertEqual(validate_rowcol_tensor_quant_gfx_arch(given), expected) + + def test_empty_means_unspecified(self): + self.assertEqual(validate_rowcol_tensor_quant_gfx_arch(""), "") + + def test_a_typo_is_rejected_rather_than_silently_gfx9(self): + for arch in ("gfx1205", "gfx12500", "gfx942x", "gfx", "1250", "GFX1250"): + with self.subTest(arch=arch): + with self.assertRaises(ValueError): + validate_rowcol_tensor_quant_gfx_arch(arch) + + def test_architectures_without_a_defensible_tile_are_rejected(self): + # These are real targets and some are listed in arch_specs.json, but neither + # default tile is correct for them: gfx1200/gfx1201 are WMMA parts with a + # 16x16x16 8-bit fragment, and gfx908 has no fp8 MFMA at all. Accepting them + # would hand back a tile this operator has never validated. + for arch in ("gfx908", "gfx1100", "gfx1200", "gfx1201", "gfx1201:xnack-"): + with self.subTest(arch=arch): + with self.assertRaises(ValueError): + validate_rowcol_tensor_quant_gfx_arch(arch) + + def test_the_error_names_the_value_and_the_alternatives(self): + with self.assertRaises(ValueError) as ctx: + validate_rowcol_tensor_quant_gfx_arch("gfx1205") + message = str(ctx.exception) + self.assertIn("gfx1205", message) + for arch in ROWCOL_TENSOR_QUANT_SUPPORTED_ARCHES: + self.assertIn(arch, message) + + def test_the_set_matches_the_bridge_runtime(self): + # One advertised support set per bridge. gemm_utils is the runtime entry + # point; if the two drift, --gfx-arch accepts a target the runner refuses, + # or refuses one it accepts. + sys.path.insert(0, str(DISPATCHER_DIR / "python")) + import gemm_utils + + self.assertEqual( + sorted(ROWCOL_TENSOR_QUANT_SUPPORTED_ARCHES), + sorted(gemm_utils._SUPPORTED_ARCHES), + ) + + if __name__ == "__main__": unittest.main() From 0e879ed02c1a7de055c29541bd50c238600b5e5b Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk <3836908+ozturkosu@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:45:28 -0400 Subject: [PATCH 16/24] fix(ck-tile): normalize the arch before validating it, not after _resolve_arch() checked its input against _SUPPORTED_ARCHES and only the callers downstream of it normalized, so a target carrying feature flags was rejected outright: gfx1250 -> gfx1250 gfx1250:xnack- -> ValueError: Unsupported GPU architecture gfx950:sramecc+:xnack- -> ValueError: Unsupported GPU architecture This branch spent several commits pushing normalization to the right boundaries and never reached this one, which is the entry point every other path goes through -- expand_sweep() calls it before anything else. Measured delta, _resolve_arch() over 15 inputs, before vs after: gfx90a / gfx942 / gfx950 / gfx1250 unchanged (pass through) gfx90a:xnack+ ValueError -> gfx90a CHANGED gfx942:sramecc+:xnack- ValueError -> gfx942 CHANGED gfx950:sramecc+:xnack- ValueError -> gfx950 CHANGED gfx1250:xnack- ValueError -> gfx1250 CHANGED gfx908 / gfx908:xnack+ / gfx1200 ValueError (unchanged) gfx942x / gfx ValueError (unchanged) "" / None unchanged (autodetect) Four of fifteen change, all in the same direction: previously-raising becomes succeeding with the base name. Nothing that used to succeed changes, and nothing that used to raise for being unsupported starts passing -- gfx908:xnack+ still raises, because normalizing feeds the gate, it does not bypass it. The error text now shows both the input and the normalized form. One correction to the finding as reported. "A real device reports the suffixed form" is true of hipDeviceProp_t::gcnArchName and of rocminfo's ISA line, but _get_arch()'s rocminfo fallback reads the agent "Name:" field, which is bare (verified: this host prints "Name: gfx90a" alongside "amdgcn-amd-amdhsa--gfx90a:sramecc+:xnack-"). So autodetection on such a host was not broken. What was actually unusable is the explicit path -- and there is a concrete source of suffixed names for it inside this repository: projects/composablekernel/CMakeLists.txt sets GPU_TARGETS to "gfx908:xnack+;gfx90a:xnack+;gfx942:xnack+;gfx950:xnack+", so a caller copying a target from the build configuration was rejected. amd-smi's TARGET_GRAPHICS_VERSION is captured as \S+ and can also carry a suffix. The suffix is dropped rather than carried into --offload-arch. That is the same target string every already-working path used, since a suffixed name could not get through at all; a caller needing a specific xnack setting must still pass it, as before. Scoped to gemm_utils.py. batched_gemm_utils.py has the identical _resolve_arch and the identical defect -- it is a different operator, not touched by this PR, and folding it in would widen the blast radius the panel is already asking to narrow. Reported, not fixed here. tests/test_gemm_utils.py: 34 -> 41 tests, 31 subtests, all passing. The normalize_gfx_arch fallback (codegen_common is not importable at gemm_utils import time) is pinned to the canonical helper on both branches. (cherry picked from commit f0af3afd34b76a01bd3a3efe8e683610e360f2db) --- .../dispatcher/python/gemm_utils.py | 69 +++++++++--- .../dispatcher/tests/test_gemm_utils.py | 102 ++++++++++++++++++ 2 files changed, 158 insertions(+), 13 deletions(-) diff --git a/projects/composablekernel/dispatcher/python/gemm_utils.py b/projects/composablekernel/dispatcher/python/gemm_utils.py index 0475a58e6a29..89505de26b1d 100644 --- a/projects/composablekernel/dispatcher/python/gemm_utils.py +++ b/projects/composablekernel/dispatcher/python/gemm_utils.py @@ -73,6 +73,26 @@ def _detect_gpu_arch_via_amd_smi() -> Optional[str]: return None +def normalize_gfx_arch(arch: str) -> str: + """Strip feature suffixes from a gfx target string. + + ``"gfx950:sramecc+:xnack-"`` -> ``"gfx950"``. Empty input passes through. + + Delegates to ``codegen_common.normalize_gfx_arch``, the dispatcher tree's + single source of truth, whenever it can be imported. It usually cannot be at + this point: nothing has put the ``codegen`` dir on ``sys.path`` yet -- only + ``ctypes_utils.get_arch_filter_data()`` does that, and it may never be called -- + so this module has to be able to answer without it, the same way it already + falls back for ``_detect_gpu_arch_via_amd_smi``. The two are pinned to identical + behaviour by tests/test_gemm_utils.py::TestArchNormalizationMatchesCodegen. + """ + try: + from codegen_common import normalize_gfx_arch as _canonical # noqa: WPS433 + except ImportError: + return arch.split(":", 1)[0] + return _canonical(arch) + + @functools.lru_cache(maxsize=1) def _get_arch() -> str: """Detect the GPU architecture from rocminfo and validate it. @@ -105,29 +125,52 @@ def _get_arch() -> str: "gfx_arch (one of " f"{', '.join(_SUPPORTED_ARCHES)})." ) - if detected not in _SUPPORTED_ARCHES: + return _validate_arch(detected) + + +def _validate_arch(arch: str) -> str: + """Normalize a gfx target, then check it against ``_SUPPORTED_ARCHES``. + + Normalization comes FIRST, and that ordering is the whole point. Both sources + that feed this -- amd-smi and rocminfo -- report the target with its feature + flags on a real device ("gfx950:sramecc+:xnack-", "gfx1250:xnack-"), and + validating that string raw rejected the only name such a device ever gives: + + gfx1250 -> ValueError + gfx950:sramecc+:xnack- -> ValueError + + Every caller wants the bare target regardless. It is stamped onto + ``GemmKernelConfig.gfx_arch``, handed to ``--offload-arch``, and used as the key + into the warp tables, none of which know about suffixes. + + The suffix is dropped, not carried through: ``--offload-arch=gfx90a`` rather + than ``gfx90a:xnack+``. That is not a regression -- it is the target string every + already-working path used, because a suffixed name could not get past this + function at all before. It does mean an xnack+ device is compiled for the bare + target; if a caller needs a specific xnack setting it has to pass the flag + itself, exactly as it had to before. + """ + base = normalize_gfx_arch(arch) + if base not in _SUPPORTED_ARCHES: raise ValueError( - f"Unsupported GPU architecture {detected!r}; supported: " - f"{', '.join(_SUPPORTED_ARCHES)}." + f"Unsupported GPU architecture {arch!r}" + + (f" (normalized to {base!r})" if base != arch else "") + + f"; supported: {', '.join(_SUPPORTED_ARCHES)}." ) - return detected + return base def _resolve_arch(arch: Optional[str]) -> str: """Resolve a possibly-``None`` arch to a validated, supported ``gfxNNN``. - ``None``/empty -> detect via :func:`_get_arch`. An explicit value is - validated against ``_SUPPORTED_ARCHES`` (raising ``ValueError`` if unknown) - so a typo can never silently reach the compiler. + ``None``/empty -> detect via :func:`_get_arch`. An explicit value is normalized + and then validated against ``_SUPPORTED_ARCHES`` (raising ``ValueError`` if + unknown) so a typo can never silently reach the compiler, while the suffixed + name a device actually reports resolves to its base. """ if not arch: return _get_arch() - if arch not in _SUPPORTED_ARCHES: - raise ValueError( - f"Unsupported GPU architecture {arch!r}; supported: " - f"{', '.join(_SUPPORTED_ARCHES)}." - ) - return arch + return _validate_arch(arch) def _cshuffle_store_ok( diff --git a/projects/composablekernel/dispatcher/tests/test_gemm_utils.py b/projects/composablekernel/dispatcher/tests/test_gemm_utils.py index bcf7fafe0cd9..af4677f04d8d 100644 --- a/projects/composablekernel/dispatcher/tests/test_gemm_utils.py +++ b/projects/composablekernel/dispatcher/tests/test_gemm_utils.py @@ -367,3 +367,105 @@ def test_grouped_gfx1250_ci_config_uses_wmma_warp_tile(self): self.assertEqual(tc["warp_tile_m"]["values"], [16]) self.assertEqual(tc["warp_tile_n"]["values"], [16]) self.assertEqual(tc["warp_tile_k"]["values"], [32]) + + +# --- feature-suffixed arch names ------------------------------------------- +# _resolve_arch() validated its input against _SUPPORTED_ARCHES *before* +# normalizing it, so a target carrying feature flags was rejected outright: +# +# gfx1250 -> ValueError +# gfx950:sramecc+:xnack- -> ValueError +# +# That is the form hipDeviceProp_t::gcnArchName reports, the form rocminfo prints +# for the ISA line, and the form this repository's own CMakeLists.txt uses for +# GPU_TARGETS ("gfx908:xnack+;gfx90a:xnack+;gfx942:xnack+;gfx950:xnack+"), so a +# caller copying a target from any of those could not drive the bridge with it. +# Normalization now runs first and the base name is validated. +from gemm_utils import _validate_arch, normalize_gfx_arch # noqa: E402 + + +class TestSuffixedArchNames(unittest.TestCase): + def test_supported_arch_with_feature_suffix_resolves_to_its_base(self): + for given, expected in [ + ("gfx90a:xnack+", "gfx90a"), + ("gfx942:xnack+", "gfx942"), + ("gfx942:sramecc+:xnack-", "gfx942"), + ("gfx950:sramecc+:xnack-", "gfx950"), + ("gfx1250:xnack-", "gfx1250"), + ]: + with self.subTest(arch=given): + self.assertEqual(_resolve_arch(given), expected) + + def test_bare_supported_arches_are_unchanged(self): + # The whole point of normalizing before validating is that it costs the + # existing paths nothing. + for arch in _SUPPORTED_ARCHES: + with self.subTest(arch=arch): + self.assertEqual(_resolve_arch(arch), arch) + + def test_an_unsupported_arch_still_raises_with_or_without_a_suffix(self): + # Normalizing must not turn the gate off: gfx908 is a real target, it is + # simply not one this bridge supports, and neither form may pass. + for arch in ("gfx908", "gfx908:xnack+", "gfx1200", "gfx1201:xnack-"): + with self.subTest(arch=arch): + with self.assertRaises(ValueError): + _resolve_arch(arch) + + def test_a_typo_still_raises(self): + for arch in ("gfx942x", "gfx1205", "gfx", "gfx950x:xnack-"): + with self.subTest(arch=arch): + with self.assertRaises(ValueError): + _resolve_arch(arch) + + def test_the_error_shows_both_the_input_and_the_normalized_form(self): + with self.assertRaises(ValueError) as ctx: + _validate_arch("gfx908:xnack+") + message = str(ctx.exception) + self.assertIn("gfx908:xnack+", message) + self.assertIn("gfx908", message) + + +class TestArchNormalizationMatchesCodegen(unittest.TestCase): + """gemm_utils cannot import codegen_common at module scope -- nothing has put + the codegen dir on sys.path by then -- so it carries a fallback. Pin the two + to identical behaviour, the same way this branch pins tile_engine's + _base_gfx_arch to codegen_common.normalize_gfx_arch.""" + + CASES = ( + "gfx942", + "gfx942:sramecc+:xnack-", + "gfx1250:xnack-", + "gfx950:sramecc+", + "gfx908", + "", + "not-an-arch", + ) + + def test_delegates_to_codegen_common_when_importable(self): + sys.path.insert(0, str(DISPATCHER_DIR / "codegen")) + from codegen_common import normalize_gfx_arch as canonical + + for arch in self.CASES: + with self.subTest(arch=arch): + self.assertEqual(normalize_gfx_arch(arch), canonical(arch)) + + def test_the_fallback_agrees_with_the_canonical_helper(self): + sys.path.insert(0, str(DISPATCHER_DIR / "codegen")) + from codegen_common import normalize_gfx_arch as canonical + + import builtins + + real_import = builtins.__import__ + + def no_codegen_common(name, *args, **kwargs): + if name == "codegen_common": + raise ImportError("simulated: codegen dir not on sys.path") + return real_import(name, *args, **kwargs) + + builtins.__import__ = no_codegen_common + try: + for arch in self.CASES: + with self.subTest(arch=arch): + self.assertEqual(normalize_gfx_arch(arch), canonical(arch)) + finally: + builtins.__import__ = real_import From 507e36335b74f4742b4ec23f4b042b061cfc79e5 Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk <3836908+ozturkosu@users.noreply.github.com> Date: Fri, 11 Sep 2026 05:04:48 -0400 Subject: [PATCH 17/24] fix(ck-tile): confine the warp-table normalization to gfx1250 This branch normalized the WARP_SUPPORTED_COMBINATIONS lookup so that a suffixed target would find its entry. That was a regression, not a fix, and it is reverted here for everything except gfx1250. dispatcher/codegen/arch_specs.json is the declared source of truth for warp maps and generates both the Python table and the C++ runtime filter. It lists seven maps for gfx942 and nine for gfx950. This table lists three. Normalizing the lookup therefore made suffixed gfx9 targets reject configurations the repository's own spec declares valid -- narrowing generation for any configure that passes a suffixed target, which the ASAN branch of projects/composablekernel/CMakeLists.txt does via CK_GPU_TARGETS "gfx908:xnack+;gfx90a:xnack+;gfx942:xnack+;gfx950:xnack+". The raw .get is restored and normalization is confined to gfx1250, which is absent from arch_specs.json entirely. For that architecture this table is the only listing, so there is no shipped behaviour to regress and the strict reading is the correct one: gfx1250 is wave32 WMMA with no 32x32 fragment, and a gfx9 warp map there compiles and returns garbage rather than failing. Measured over {gfx90a, gfx942, gfx950, gfx1201, gfx1250} x {bare, :xnack-, :sramecc+:xnack-} x all 64 warp maps, against this commit's parent: gfx90a suffixed (both forms) 61 decisions change each gfx942 suffixed (both forms) 61 decisions change each gfx950 suffixed (both forms) 61 decisions change each gfx1201 suffixed (both forms) 60 decisions change each gfx1250:xnack- 0 decisions change every bare target 0 decisions change Note the direction: the suffixed gfx9 targets are what move, because restoring the permissive fallthrough for them is precisely how the regression is removed. gfx1250 does not move here -- it was already normalized and stays that way. An earlier draft of this message stated the opposite, naming gfx1250:xnack- as the only target affected; that was inverted and is corrected above. Against develop, after this change: suffixed gfx90a/gfx942/gfx950/gfx1201 are back to 0 changed decisions, and the only remaining delta is gfx1250 -- 5 on the bare name and 55 on gfx1250:xnack- -- which is this branch's intended restriction. Tests: 144 passed, 384 subtests. --- ...test_gemm_validation_arch_normalization.py | 89 +++++++++++++++++++ .../ops/gemm/gemm_validation_utils.py | 40 +++++++-- 2 files changed, 123 insertions(+), 6 deletions(-) diff --git a/projects/composablekernel/dispatcher/tests/test_gemm_validation_arch_normalization.py b/projects/composablekernel/dispatcher/tests/test_gemm_validation_arch_normalization.py index 053076442b8b..b9167c7fcbf4 100644 --- a/projects/composablekernel/dispatcher/tests/test_gemm_validation_arch_normalization.py +++ b/projects/composablekernel/dispatcher/tests/test_gemm_validation_arch_normalization.py @@ -122,6 +122,95 @@ def test_gfx1250_rejects_the_32x32_warp_tile_bare_and_suffixed(self): self.assertFalse(ok) +@unittest.skipIf(gvu is None, "tile_engine tree not present") +class TestWarpConfigurationLookupIsUnchangedOnGfx9(unittest.TestCase): + """The WARP_SUPPORTED_COMBINATIONS lookup stays raw for gfx9. On purpose. + + Normalizing it would activate a validator that a suffixed target currently + bypasses -- which sounds like a fix until you look at what the table + contains. dispatcher/codegen/arch_specs.json, the declared single JSON + source of truth for arch data, lists seven warp maps for gfx942 and nine for + gfx950; this table lists three for each. Turning it on for suffixed names + therefore rejects configurations the repo's own spec calls valid. + + gfx1250 is the exception, and the only one: it is absent from + arch_specs.json entirely, so this table is its sole listing, and it is the + target this work exists for. + + These tests fail if someone "tidies up" the asymmetry. + """ + + _GRID = [1, 2, 4, 8] + + def _decisions(self, target): + return { + (m, n, k): gvu.validate_warp_configuration(m, n, k, target) + for m, n, k in itertools.product(self._GRID, repeat=3) + } + + def test_suffixed_gfx9_targets_stay_permissive(self): + # develop's behaviour: an unrecognized key logs and allows. Pinned so the + # gfx9 instance sets cannot move under an ASAN-style configure. + for bare in ("gfx90a", "gfx942", "gfx950"): + suffixed = _SUFFIXES[bare] + with self.subTest(arch=suffixed): + self.assertTrue( + all(self._decisions(suffixed).values()), + f"{suffixed} must remain unchecked by this table", + ) + + def test_bare_gfx9_targets_still_enforce_their_three_maps(self): + for bare in ("gfx90a", "gfx942", "gfx950"): + decisions = self._decisions(bare) + with self.subTest(arch=bare): + self.assertEqual( + {c for c, ok in decisions.items() if ok}, + {(1, 4, 1), (2, 2, 1), (4, 1, 1)}, + ) + + def test_the_table_is_a_strict_subset_of_arch_specs_json(self): + """Why the gfx9 lookup is left alone, asserted rather than asserted-in-prose. + + If the two ever agree, the argument above evaporates and the raw lookup + should be revisited -- so this test is written to fail at that point. + """ + import json + + specs_path = DISPATCHER_DIR / "codegen" / "arch_specs.json" + if not specs_path.exists(): + self.skipTest("arch_specs.json not present") + specs = json.load(open(specs_path))["architectures"] + for arch in ("gfx942", "gfx950"): + table = {tuple(c) for c in gvu.WARP_SUPPORTED_COMBINATIONS[arch]} + spec = {tuple(c) for c in specs[arch]["warp_configs"]} + with self.subTest(arch=arch): + self.assertTrue( + table < spec, + f"{arch}: gemm_validation_utils lists {sorted(table)}, " + f"arch_specs.json lists {sorted(spec)}. If these now agree, " + f"normalizing the gfx9 lookup is no longer a coverage loss.", + ) + + def test_suffixed_gfx1250_is_checked_like_bare_gfx1250(self): + self.assertEqual( + self._decisions(_SUFFIXES["gfx1250"]), self._decisions("gfx1250") + ) + + def test_suffixed_gfx1250_rejects_maps_absent_from_the_table(self): + # develop allowed all of these on "gfx1250:xnack-" because the lookup + # missed. 1x1x1 is the example the review asked about. + for combo in ((1, 1, 1), (1, 1, 2), (8, 8, 8), (4, 4, 1)): + with self.subTest(combo=combo): + self.assertFalse( + gvu.validate_warp_configuration(*combo, _SUFFIXES["gfx1250"]) + ) + + def test_gfx1201_suffix_is_not_swept_up_by_the_gfx1250_exception(self): + # The exception is exact-gfx1250, not gfx12-family. gfx1201 keeps + # develop's permissive behaviour under a suffix. + self.assertTrue(all(self._decisions(_SUFFIXES["gfx1201"]).values())) + + if __name__ == "__main__": logging.disable(logging.CRITICAL) sys.exit(unittest.main()) diff --git a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py index 1c29d6a884bf..a5bcf5e8d0b4 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py +++ b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py @@ -347,12 +347,40 @@ def validate_warp_configuration( current_combination = [warp_m, warp_n, warp_k] - # Normalize first: a device reports "gfx1250:xnack-", and an unrecognized key - # falls through permissively below, which would disable this restriction on the - # very target it exists for. - allowed_combinations = WARP_SUPPORTED_COMBINATIONS.get( - gpu_name.split(":")[0] if gpu_name else gpu_name, {} - ) + # Deliberately a RAW lookup. Unlike every other arch test in this module, this + # one is NOT normalized, and that asymmetry is the point. + # + # Normalizing it looks like an obvious fix -- a suffixed name misses the table, + # an empty dict comes back, and the permissive branch below turns the whole + # restriction off. But this table is not a description of the hardware. It is a + # hand-maintained subset, and it is stale: dispatcher/codegen/arch_specs.json, + # the declared single JSON source of truth feeding both the Python codegen and + # the C++ runtime filter, lists seven warp maps for gfx942 + # ([1,1,1] [1,2,1] [1,4,1] [2,1,1] [2,1,2] [2,2,1] [4,1,1]) and nine for gfx950, + # where this table lists three. Switching a suffixed gfx942 from "unchecked" to + # "checked against three of its seven maps" does not reject invalid + # configurations; it rejects valid ones, silently narrowing instance generation + # for the ASAN configure in projects/composablekernel/CMakeLists.txt, which sets + # GPU_TARGETS to "gfx908:xnack+;gfx90a:xnack+;gfx942:xnack+;gfx950:xnack+". + # + # Second reason, independent of the first: dispatcher/python/gemm_utils.py + # deliberately mirrors this function ("Unknown arch => permissive (matches + # Old-TE's log-and-allow behavior)") so that the bridge and Old-TE emit the same + # instance set. It does a raw .get too. Making this side strict for suffixed + # names and not that one diverges the two sets, which is the exact failure that + # comment exists to prevent. + # + # Widening the table to match arch_specs.json would be a real fix, but it + # changes what seven instance builders generate on their primary targets and + # belongs in its own PR with its own build evidence -- not smuggled in as a + # side effect of gfx1250 enablement. + # + # The one place normalization is needed is gfx1250, the target this branch + # exists for and one that arch_specs.json does not describe at all, so this + # table is its only listing. Scope the change to exactly that. + allowed_combinations = WARP_SUPPORTED_COMBINATIONS.get(gpu_name, {}) + if not allowed_combinations and _base_gfx_arch(gpu_name) == "gfx1250": + allowed_combinations = WARP_SUPPORTED_COMBINATIONS.get("gfx1250", {}) if not allowed_combinations: # If GPU not recognized, try to be permissive but log warning logging.warning(f"No warp_[m/n/k] combinations found for GPU: {gpu_name}") From b7a262ffa7a688329c4dd7ee220fc88a3b9c9348 Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk <3836908+ozturkosu@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:42:16 -0400 Subject: [PATCH 18/24] docs(ck-tile): drop a false justification for the raw warp-table lookup The comment defending the unnormalized WARP_SUPPORTED_COMBINATIONS lookup gave two reasons. The second one is not true. It claimed that dispatcher/python/gemm_utils.py mirrors this function with a raw .get, so normalizing here and not there would diverge the two instance sets. But the mirror cannot observe the difference. _warp_config_supported() has exactly one call site, expand_sweep(), and expand_sweep() sets `arch = _resolve_arch(arch)` before it -- and _resolve_arch() raises ValueError for any name outside ("gfx90a", "gfx942", "gfx950", "gfx1250"). A suffixed target never reaches the mirror's .get; it fails earlier. Whatever this side does with suffixed names, the two sets cannot diverge on them. Deleted rather than reworded. A reason that does not hold is worse than no reason: it makes the decision look doubly supported and invites the next reader to maintain a constraint that does not exist. The first reason carries the argument by itself and was re-verified against arch_specs.json at this head -- gfx942 has seven warp_configs, gfx950 nine, this table three -- so normalizing a suffixed gfx942 would reject valid configurations, not invalid ones. Comment-only. No test asserted the removed text. (cherry picked from commit 5599a9f75a3d7b345cc4451ea4eadb51a719db93) --- .../tile_engine/ops/gemm/gemm_validation_utils.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py index a5bcf5e8d0b4..1ebf8ed806be 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py +++ b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py @@ -363,13 +363,6 @@ def validate_warp_configuration( # for the ASAN configure in projects/composablekernel/CMakeLists.txt, which sets # GPU_TARGETS to "gfx908:xnack+;gfx90a:xnack+;gfx942:xnack+;gfx950:xnack+". # - # Second reason, independent of the first: dispatcher/python/gemm_utils.py - # deliberately mirrors this function ("Unknown arch => permissive (matches - # Old-TE's log-and-allow behavior)") so that the bridge and Old-TE emit the same - # instance set. It does a raw .get too. Making this side strict for suffixed - # names and not that one diverges the two sets, which is the exact failure that - # comment exists to prevent. - # # Widening the table to match arch_specs.json would be a real fix, but it # changes what seven instance builders generate on their primary targets and # belongs in its own PR with its own build evidence -- not smuggled in as a From 742fe7f10f4541cad8ffc3a29ff31fc86ec7a775 Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk <3836908+ozturkosu@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:47:51 -0400 Subject: [PATCH 19/24] docs(ck-tile): correct where suffixed gfx names actually come from The _validate_arch docstring justified normalize-before-validate by claiming amd-smi and rocminfo "report the target with its feature flags on a real device" and that the suffixed form was "the only name such a device ever gives". Both are false, measured on a gfx90a device: amd-smi TARGET_GRAPHICS_VERSION, rocm_agent_enumerator and the rocminfo agent Name: line all return a bare gfx90a. _get_arch cannot even emit a suffixed name -- rocminfo's only suffixed string is the ISA line amdgcn-amd-amdhsa--gfx90a:sramecc+:xnack-, which fails the startswith("gfx") check after the split. The ordering change is still correct; the reason is different. Suffixed names reach the function from callers: this repo's CMakeLists.txt ASAN branch sets CK_GPU_TARGETS "gfx908:xnack+;gfx90a:xnack+;gfx942:xnack+;gfx950:xnack+", and hipDeviceProp_t::gcnArchName returns gfx90a:sramecc+:xnack- (measured). Either can be copied into --gfx-arch and arrives here unmodified. Also drop the false "gfx1250 -> ValueError" row from the before/after table in both the docstring and the mirrored comment in tests/test_gemm_utils.py. gfx1250 is in _SUPPORTED_ARCHES and always passed, as test_bare_supported_arches_are_unchanged asserts directly below that comment. Comment-only; no behaviour change. (cherry picked from commit 57fd4efcefef16af01e6edc8f48dcc6a47776a01) --- .../dispatcher/python/gemm_utils.py | 30 +++++++++++++++---- .../dispatcher/tests/test_gemm_utils.py | 6 ++-- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/projects/composablekernel/dispatcher/python/gemm_utils.py b/projects/composablekernel/dispatcher/python/gemm_utils.py index 89505de26b1d..b81b162f5c66 100644 --- a/projects/composablekernel/dispatcher/python/gemm_utils.py +++ b/projects/composablekernel/dispatcher/python/gemm_utils.py @@ -131,14 +131,34 @@ def _get_arch() -> str: def _validate_arch(arch: str) -> str: """Normalize a gfx target, then check it against ``_SUPPORTED_ARCHES``. - Normalization comes FIRST, and that ordering is the whole point. Both sources - that feed this -- amd-smi and rocminfo -- report the target with its feature - flags on a real device ("gfx950:sramecc+:xnack-", "gfx1250:xnack-"), and - validating that string raw rejected the only name such a device ever gives: + Normalization comes FIRST, and that ordering is the whole point -- but not + because autodetection produces suffixes. It does not. On a real device all + three autodetect sources report the BARE target: ``amd-smi``'s + ``TARGET_GRAPHICS_VERSION``, ``rocm_agent_enumerator``, and the agent + ``Name:`` line that ``_get_arch`` above parses all print ``gfx90a`` + (measured on a gfx90a device). ``_get_arch`` could not return a suffixed + name even if rocminfo offered one: the only suffixed string in that output + is the ISA line, ``amdgcn-amd-amdhsa--gfx90a:sramecc+:xnack-``, which fails + the ``startswith("gfx")`` check after the split. + + Suffixed names get here from CALLERS, not from detection, and they are real: + + * this repository's own CMakeLists.txt sets, on the ASAN branch, + ``CK_GPU_TARGETS "gfx908:xnack+;gfx90a:xnack+;gfx942:xnack+;gfx950:xnack+"``; + * ``hipDeviceProp_t::gcnArchName`` returns ``gfx90a:sramecc+:xnack-`` + (measured on the same device). + + Either is a plausible thing to copy into ``--gfx-arch`` or an explicit + ``arch=``, and that value reaches this function unmodified. Validating it raw + rejected it: - gfx1250 -> ValueError gfx950:sramecc+:xnack- -> ValueError + Bare targets were never the problem; they passed before this change and pass + now (tests/test_gemm_utils.py::test_bare_supported_arches_are_unchanged). + Normalizing first costs a supported bare name nothing and makes the suffixed + spellings above usable. + Every caller wants the bare target regardless. It is stamped onto ``GemmKernelConfig.gfx_arch``, handed to ``--offload-arch``, and used as the key into the warp tables, none of which know about suffixes. diff --git a/projects/composablekernel/dispatcher/tests/test_gemm_utils.py b/projects/composablekernel/dispatcher/tests/test_gemm_utils.py index af4677f04d8d..57d7e8f5147e 100644 --- a/projects/composablekernel/dispatcher/tests/test_gemm_utils.py +++ b/projects/composablekernel/dispatcher/tests/test_gemm_utils.py @@ -373,10 +373,12 @@ def test_grouped_gfx1250_ci_config_uses_wmma_warp_tile(self): # _resolve_arch() validated its input against _SUPPORTED_ARCHES *before* # normalizing it, so a target carrying feature flags was rejected outright: # -# gfx1250 -> ValueError # gfx950:sramecc+:xnack- -> ValueError # -# That is the form hipDeviceProp_t::gcnArchName reports, the form rocminfo prints +# Bare targets were unaffected -- gfx1250 is in _SUPPORTED_ARCHES and passed both +# before and after (test_bare_supported_arches_are_unchanged below). +# +# The suffixed form is what hipDeviceProp_t::gcnArchName reports, the form rocminfo prints # for the ISA line, and the form this repository's own CMakeLists.txt uses for # GPU_TARGETS ("gfx908:xnack+;gfx90a:xnack+;gfx942:xnack+;gfx950:xnack+"), so a # caller copying a target from any of those could not drive the bridge with it. From 728a208c076c53e059f4ed710546b054ee73c673 Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk <3836908+ozturkosu@users.noreply.github.com> Date: Fri, 18 Sep 2026 01:07:10 -0400 Subject: [PATCH 20/24] fix(ck): address grouped quant architecture and ABI review feedback Reject config/build target mismatches before code generation. Embed explicit codegen targets in kernel headers and compare them with GFX_ARCH at compile time, covering both gfx1250-to-gfx9 and gfx9-to-gfx1250 mismatches. Normalize explicit and inferred CMake architecture values. Limit quant operator validation to gfx942, gfx950, and gfx1250. Match top-level CK's compiler target-ID and gfx1250 legacy WarpGemm definitions in standalone Python and ctypes builds. Report missing TileN/PadN exports as an actionable ABI error and only clean up a successfully initialized library. Advance the cache revision to abi3 so older binaries are rebuilt with the new generated-target checks. Add 41 host regression cases for target matching, support validation, CMake arguments, old-library loading, and the production C++ M/N guards. Extend existing cache tests to cover abi2 artifacts. Validation: - 341 tests and 941 subtests passed locally. - All 8 selected CTest suites passed after conflict resolution. - Real MI400 RowColQuant and TensorQuant FP8/BF8 smoke checks passed on gfx1250 with ROCm 10.0 (reported max relative error about 0.0005). --- .../dispatcher/bindings/ctypes/CMakeLists.txt | 32 +++- .../grouped_gemm_rowcolquant_ctypes_lib.cpp | 28 ++- .../grouped_gemm_tensorquant_ctypes_lib.cpp | 28 ++- .../dispatcher/codegen/codegen_common.py | 13 +- ...nified_grouped_gemm_rowcolquant_codegen.py | 14 +- ...nified_grouped_gemm_tensorquant_codegen.py | 14 +- .../python/grouped_gemm_rowcolquant_utils.py | 47 +++-- .../python/grouped_gemm_tensorquant_utils.py | 47 +++-- .../dispatcher/tests/CMakeLists.txt | 5 +- .../dispatcher/tests/test_codegen_common.py | 13 +- .../test_grouped_gemm_rowcolquant_utils.py | 12 +- .../test_grouped_gemm_tensorquant_utils.py | 12 +- .../tests/test_quant_review_regressions.py | 178 ++++++++++++++++++ 13 files changed, 335 insertions(+), 108 deletions(-) create mode 100644 projects/composablekernel/dispatcher/tests/test_quant_review_regressions.py diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/CMakeLists.txt b/projects/composablekernel/dispatcher/bindings/ctypes/CMakeLists.txt index 9e5189e3784b..dd25d4f539ff 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/CMakeLists.txt +++ b/projects/composablekernel/dispatcher/bindings/ctypes/CMakeLists.txt @@ -351,16 +351,22 @@ if(ROWCOLQUANT_GEMM_KERNEL_HEADERS) else() set(_rowcolquant_gfx_arch "gfx942") endif() - # Strip feature suffixes ("gfx942:sramecc+:xnack-"). CMAKE_HIP_ARCHITECTURES - # carries them for ASAN configures, and GFX_ARCH is compared against - # kSupportedArchs and the compile-time tile guards in the .cpp, which want the - # bare target. Mirrors normalize_gfx_arch() on the Python side. - string(REGEX REPLACE ":.*$" "" _rowcolquant_gfx_arch "${_rowcolquant_gfx_arch}") set(CK_TILE_ROWCOLQUANT_GFX_ARCH "${_rowcolquant_gfx_arch}" CACHE STRING "GFX arch for the CMake-built RowColQuant ctypes .so (default: first CMAKE_HIP_ARCHITECTURES or gfx942)") endif() + # Normalize both inferred and explicitly supplied cache values. + string(REGEX REPLACE ":.*$" "" CK_TILE_ROWCOLQUANT_GFX_ARCH "${CK_TILE_ROWCOLQUANT_GFX_ARCH}") message(STATUS "RowColQuant ctypes lib GFX_ARCH: ${CK_TILE_ROWCOLQUANT_GFX_ARCH}") + # Standalone dispatcher builds do not inherit the top-level CK definitions. + # Override the directory-wide ID for this single-target library. + string(REGEX REPLACE "^gfx" "0x" _rowcolquant_target_id "${CK_TILE_ROWCOLQUANT_GFX_ARCH}") + target_compile_options(dispatcher_grouped_gemm_rowcolquant_lib PRIVATE + -UCK_CMAKE_GPU_TARGET_IDS "-DCK_CMAKE_GPU_TARGET_IDS=${_rowcolquant_target_id}") + if(CK_TILE_ROWCOLQUANT_GFX_ARCH STREQUAL "gfx1250") + target_compile_definitions(dispatcher_grouped_gemm_rowcolquant_lib PRIVATE USE_NEW_UNIFIED_FRAMEWORK=0) + endif() + target_compile_definitions(dispatcher_grouped_gemm_rowcolquant_lib PRIVATE CK_TILE_SINGLE_KERNEL_INCLUDE GFX_ARCH="${CK_TILE_ROWCOLQUANT_GFX_ARCH}" @@ -405,16 +411,22 @@ if(TENSORQUANT_GEMM_KERNEL_HEADERS) else() set(_tensorquant_gfx_arch "gfx942") endif() - # Strip feature suffixes ("gfx942:sramecc+:xnack-"). CMAKE_HIP_ARCHITECTURES - # carries them for ASAN configures, and GFX_ARCH is compared against - # kSupportedArchs and the compile-time tile guards in the .cpp, which want the - # bare target. Mirrors normalize_gfx_arch() on the Python side. - string(REGEX REPLACE ":.*$" "" _tensorquant_gfx_arch "${_tensorquant_gfx_arch}") set(CK_TILE_TENSORQUANT_GFX_ARCH "${_tensorquant_gfx_arch}" CACHE STRING "GFX arch for the CMake-built TensorQuant ctypes .so (default: first CMAKE_HIP_ARCHITECTURES or gfx942)") endif() + # Normalize both inferred and explicitly supplied cache values. + string(REGEX REPLACE ":.*$" "" CK_TILE_TENSORQUANT_GFX_ARCH "${CK_TILE_TENSORQUANT_GFX_ARCH}") message(STATUS "TensorQuant ctypes lib GFX_ARCH: ${CK_TILE_TENSORQUANT_GFX_ARCH}") + # Standalone dispatcher builds do not inherit the top-level CK definitions. + # Override the directory-wide ID for this single-target library. + string(REGEX REPLACE "^gfx" "0x" _tensorquant_target_id "${CK_TILE_TENSORQUANT_GFX_ARCH}") + target_compile_options(dispatcher_grouped_gemm_tensorquant_lib PRIVATE + -UCK_CMAKE_GPU_TARGET_IDS "-DCK_CMAKE_GPU_TARGET_IDS=${_tensorquant_target_id}") + if(CK_TILE_TENSORQUANT_GFX_ARCH STREQUAL "gfx1250") + target_compile_definitions(dispatcher_grouped_gemm_tensorquant_lib PRIVATE USE_NEW_UNIFIED_FRAMEWORK=0) + endif() + target_compile_definitions(dispatcher_grouped_gemm_tensorquant_lib PRIVATE CK_TILE_SINGLE_KERNEL_INCLUDE GFX_ARCH="${CK_TILE_TENSORQUANT_GFX_ARCH}" diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp index 0c8510e43376..c21596c4408c 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp +++ b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp @@ -89,27 +89,23 @@ static bool is_supported_arch(const std::string& arch) } // --------------------------------------------------------------------------- -// Compile-time gfx1250 tile guard. -// -// The runtime helper in grouped_gemm_*_utils.py picks an arch-correct tile, but the -// CMake/codegen path does not: unified_grouped_gemm_*_codegen.py::_default_config() -// takes no arch and hard-codes the gfx9 tile. Listing gfx1250 in kSupportedArchs above -// therefore means a CMake-built gfx1250 library would initialize cleanly and then run a -// tile that does not exist on the device -- which compiles and returns garbage rather -// than failing. -// -// Rather than trust the build to pick correctly, refuse to produce such a library at -// all. gfx1250 is wave32 with RDNA-style WMMA: the only 8-bit fragments are 16x16x64 -// and 16x16x128 (there is no 32x32 WMMA), and a block of more than four warps has no -// launchable kernel entry. These are the same rules the Python validator enforces. -// -// The test is exact-gfx1250, not gfx12-family: gfx1200/gfx1201 have a different 8-bit -// fragment (16x16x16) and are not in kSupportedArchs anyway. +// Generated-target and tile guards. Codegen records an explicit target in the +// header; a build for another processor must fail before it can run that tile. +// Headers generated without an explicit target retain the gfx9 default behavior. static constexpr bool ct_starts_with(const char* s, const char* prefix) { return *prefix == '\0' ? true : (*s == *prefix && ct_starts_with(s + 1, prefix + 1)); } +static constexpr bool ct_equal(const char* a, const char* b) +{ + return *a == *b && (*a == '\0' || ct_equal(a + 1, b + 1)); +} + +static_assert(SelectedKernel::GfxArch[0] == '\0' || ct_equal(SelectedKernel::GfxArch, GFX_ARCH), + "Generated kernel architecture does not match GFX_ARCH. Regenerate the " + "header with --gfx-arch matching the build target."); + static constexpr bool kCompiledForGfx1250 = ct_starts_with(GFX_ARCH, "gfx1250"); // Family-wide companion for the M%4 guard below. That defect is an unpadded AQ view in diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp index f0b7a6b52ed6..b495cb3b4749 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp +++ b/projects/composablekernel/dispatcher/bindings/ctypes/grouped_gemm_tensorquant_ctypes_lib.cpp @@ -92,27 +92,23 @@ static bool is_supported_arch(const std::string& arch) } // --------------------------------------------------------------------------- -// Compile-time gfx1250 tile guard. -// -// The runtime helper in grouped_gemm_*_utils.py picks an arch-correct tile, but the -// CMake/codegen path does not: unified_grouped_gemm_*_codegen.py::_default_config() -// takes no arch and hard-codes the gfx9 tile. Listing gfx1250 in kSupportedArchs above -// therefore means a CMake-built gfx1250 library would initialize cleanly and then run a -// tile that does not exist on the device -- which compiles and returns garbage rather -// than failing. -// -// Rather than trust the build to pick correctly, refuse to produce such a library at -// all. gfx1250 is wave32 with RDNA-style WMMA: the only 8-bit fragments are 16x16x64 -// and 16x16x128 (there is no 32x32 WMMA), and a block of more than four warps has no -// launchable kernel entry. These are the same rules the Python validator enforces. -// -// The test is exact-gfx1250, not gfx12-family: gfx1200/gfx1201 have a different 8-bit -// fragment (16x16x16) and are not in kSupportedArchs anyway. +// Generated-target and tile guards. Codegen records an explicit target in the +// header; a build for another processor must fail before it can run that tile. +// Headers generated without an explicit target retain the gfx9 default behavior. static constexpr bool ct_starts_with(const char* s, const char* prefix) { return *prefix == '\0' ? true : (*s == *prefix && ct_starts_with(s + 1, prefix + 1)); } +static constexpr bool ct_equal(const char* a, const char* b) +{ + return *a == *b && (*a == '\0' || ct_equal(a + 1, b + 1)); +} + +static_assert(SelectedKernel::GfxArch[0] == '\0' || ct_equal(SelectedKernel::GfxArch, GFX_ARCH), + "Generated kernel architecture does not match GFX_ARCH. Regenerate the " + "header with --gfx-arch matching the build target."); + static constexpr bool kCompiledForGfx1250 = ct_starts_with(GFX_ARCH, "gfx1250"); static_assert(!kCompiledForGfx1250 || SelectedKernel::WarpTileM == 16, diff --git a/projects/composablekernel/dispatcher/codegen/codegen_common.py b/projects/composablekernel/dispatcher/codegen/codegen_common.py index 090ad01ad4d8..0ae10e1785b5 100644 --- a/projects/composablekernel/dispatcher/codegen/codegen_common.py +++ b/projects/composablekernel/dispatcher/codegen/codegen_common.py @@ -1040,16 +1040,9 @@ def rowcol_tensor_quant_default_tile(gfx_arch: str = "") -> dict: return dict(ROWCOL_TENSOR_QUANT_DEFAULT_TILE) -# Architectures the RowColQuant/TensorQuant codegen will accept on its command line. -# Deliberately the same four as dispatcher/python/gemm_utils.py::_SUPPORTED_ARCHES, -# so the bridge does not advertise one supported set on one entry point and a -# different one on another. -# -# Not taken from codegen/arch_specs.json: that file does not list gfx1250 at all, and -# it does list gfx908/gfx1100/gfx1200/gfx1201, none of which these two operators have -# a defensible default tile for -- gfx1200/gfx1201 in particular are WMMA parts whose -# 8-bit fragment is 16x16x16, so both tiles above are wrong for them. -ROWCOL_TENSOR_QUANT_SUPPORTED_ARCHES = ("gfx90a", "gfx942", "gfx950", "gfx1250") +# Operator-specific support: both bridges require native FP8/BF8. gfx90a +# belongs to generic GEMM support, but cannot initialize these quant bridges. +ROWCOL_TENSOR_QUANT_SUPPORTED_ARCHES = ("gfx942", "gfx950", "gfx1250") def validate_rowcol_tensor_quant_gfx_arch(gfx_arch: str) -> str: diff --git a/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_rowcolquant_codegen.py b/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_rowcolquant_codegen.py index 61236b0501ee..7350b22f231f 100644 --- a/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_rowcolquant_codegen.py +++ b/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_rowcolquant_codegen.py @@ -145,6 +145,7 @@ class RowColQuantKernelSpec: tile: RowColQuantTileConfig block_size: int = 256 k_block_per_cu: int = 1 + gfx_arch: str = "" @property def name(self) -> str: @@ -254,6 +255,8 @@ def generate(self, spec: RowColQuantKernelSpec) -> str: using BQDataType = {ns}::BQDataType; using AccDataType = {ns}::AccDataType; + static constexpr const char* GfxArch = "{validate_rowcol_tensor_quant_gfx_arch(spec.gfx_arch)}"; + static constexpr ck_tile::index_t TileM = {t.tile_m}; static constexpr ck_tile::index_t TileN = {t.tile_n}; static constexpr ck_tile::index_t TileK = {t.tile_k}; @@ -546,12 +549,15 @@ def generate_kernels( ) -> List[Path]: """Generate all RowColQuant kernel headers into output_dir. Returns list of generated .hpp paths. - `gfx_arch` only selects the built-in default tile and is ignored when `config` - is supplied -- an explicit config already carries its own tile_configs. + `gfx_arch` selects the default tile and is embedded in every header so the + bridge can reject a mismatched build target. Explicit tiles remain unchanged. """ + gfx_arch = validate_rowcol_tensor_quant_gfx_arch(gfx_arch) output_dir.mkdir(parents=True, exist_ok=True) cfg = config or _default_config(gfx_arch) specs = _build_specs(cfg) + for spec in specs: + spec.gfx_arch = gfx_arch if not specs: log.warning("No kernel specs produced from config — check dtypes and tile_configs") @@ -610,8 +616,8 @@ def main() -> int: help="Print kernel names that would be generated and exit") parser.add_argument("--gfx-arch", type=str, default="", help="GPU target the generated kernels will be compiled for, " - "e.g. gfx1250. Selects the built-in default tile; ignored " - "when --config/--config-json supplies tile_configs. " + "e.g. gfx1250. Embedded in generated headers; selects the default tile " + "unless --config/--config-json supplies tile_configs. " "Defaults to the gfx9 MFMA tile.") args = parser.parse_args() diff --git a/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_tensorquant_codegen.py b/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_tensorquant_codegen.py index c32125da35c4..4031559c50af 100644 --- a/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_tensorquant_codegen.py +++ b/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_tensorquant_codegen.py @@ -145,6 +145,7 @@ class TensorQuantKernelSpec: tile: TensorQuantTileConfig block_size: int = 256 k_block_per_cu: int = 1 + gfx_arch: str = "" @property def name(self) -> str: @@ -254,6 +255,8 @@ def generate(self, spec: TensorQuantKernelSpec) -> str: using BQDataType = {ns}::BQDataType; using AccDataType = {ns}::AccDataType; + static constexpr const char* GfxArch = "{validate_rowcol_tensor_quant_gfx_arch(spec.gfx_arch)}"; + static constexpr ck_tile::index_t TileM = {t.tile_m}; static constexpr ck_tile::index_t TileN = {t.tile_n}; static constexpr ck_tile::index_t TileK = {t.tile_k}; @@ -546,12 +549,15 @@ def generate_kernels( ) -> List[Path]: """Generate all TensorQuant kernel headers into output_dir. Returns list of generated .hpp paths. - `gfx_arch` only selects the built-in default tile and is ignored when `config` - is supplied -- an explicit config already carries its own tile_configs. + `gfx_arch` selects the default tile and is embedded in every header so the + bridge can reject a mismatched build target. Explicit tiles remain unchanged. """ + gfx_arch = validate_rowcol_tensor_quant_gfx_arch(gfx_arch) output_dir.mkdir(parents=True, exist_ok=True) cfg = config or _default_config(gfx_arch) specs = _build_specs(cfg) + for spec in specs: + spec.gfx_arch = gfx_arch if not specs: log.warning("No kernel specs produced from config — check dtypes and tile_configs") @@ -610,8 +616,8 @@ def main() -> int: help="Print kernel names that would be generated and exit") parser.add_argument("--gfx-arch", type=str, default="", help="GPU target the generated kernels will be compiled for, " - "e.g. gfx1250. Selects the built-in default tile; ignored " - "when --config/--config-json supplies tile_configs. " + "e.g. gfx1250. Embedded in generated headers; selects the default tile " + "unless --config/--config-json supplies tile_configs. " "Defaults to the gfx9 MFMA tile.") args = parser.parse_args() diff --git a/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py b/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py index c6c6153199be..af81d7b400d9 100644 --- a/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py +++ b/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py @@ -52,6 +52,7 @@ ROWCOL_TENSOR_QUANT_DEFAULT_TRAITS, rowcol_tensor_quant_default_tile, normalize_gfx_arch, + validate_rowcol_tensor_quant_gfx_arch, make_rowcolquant_kernel_name, ) @@ -73,7 +74,8 @@ # than papered over with a runtime fallback. # 1 -> original export set # 2 -> added dispatcher_get_tile_n() / dispatcher_get_pad_n() -_SO_ABI = 2 +# 3 -> require regenerated headers with build-target validation +_SO_ABI = 3 # ============================================================================= @@ -214,7 +216,7 @@ class RowColQuantDispatcherLib: def __init__(self, so_path: Path): self.so_path = Path(so_path) - self._cleaned_up = False + self._cleaned_up = True if not self.so_path.exists(): raise FileNotFoundError(f"RowColQuant .so not found: {self.so_path}") self._lib = ctypes.CDLL(str(self.so_path)) @@ -222,6 +224,7 @@ def __init__(self, so_path: Path): rc = self._lib.dispatcher_initialize() if rc != 0: raise RuntimeError(f"dispatcher_initialize() returned {rc}") + self._cleaned_up = False def _setup(self): lib = self._lib @@ -256,16 +259,17 @@ def _setup(self): lib.dispatcher_get_kernel_count.restype = ctypes.c_int lib.dispatcher_get_kernel_count.argtypes = [] - # TileN / pad_n are compile-time properties of the single force-included - # kernel. They are read back rather than hardcoded so the N-divisibility - # diagnostics below stay correct if the default tile changes. No missing- - # symbol fallback: this .so is always compiled from the ctypes source that - # ships these exports, by this module. - lib.dispatcher_get_tile_n.restype = ctypes.c_int - lib.dispatcher_get_tile_n.argtypes = [] - - lib.dispatcher_get_pad_n.restype = ctypes.c_int - lib.dispatcher_get_pad_n.argtypes = [] + # Direct callers can supply libraries outside the versioned cache. + try: + lib.dispatcher_get_tile_n.restype = ctypes.c_int + lib.dispatcher_get_tile_n.argtypes = [] + lib.dispatcher_get_pad_n.restype = ctypes.c_int + lib.dispatcher_get_pad_n.argtypes = [] + except AttributeError as exc: + raise RuntimeError( + f"Incompatible quant bridge ABI in {self.so_path}: missing TileN/PadN " + "exports (ABI 2 or newer required). Rebuild the library." + ) from exc lib.dispatcher_cleanup.restype = None lib.dispatcher_cleanup.argtypes = [] @@ -545,6 +549,7 @@ def _generate_rowcolquant_kernel( str(_CODEGEN_SCRIPT), "--output-dir", str(output_dir), "--config-json", config_json, + "--gfx-arch", validate_rowcol_tensor_quant_gfx_arch(config.gfx_arch), ] try: @@ -583,7 +588,7 @@ def _compile_rowcolquant_kernel( # defines below and the --offload-arch/-DGFX_ARCH we hand to hipcc -- sees the # bare target. A caller-supplied "gfx1250:xnack-" must not reach the compiler # flags. - gfx_arch = normalize_gfx_arch(gfx_arch) + gfx_arch = validate_rowcol_tensor_quant_gfx_arch(gfx_arch) ck_include = _get_ck_include_dir() static_lib = _get_dispatcher_static_lib() @@ -591,6 +596,10 @@ def _compile_rowcolquant_kernel( obj_path = so_path.with_suffix(".o") arch_defines = [] + # Match the top-level CK CMake policy in both host and device compilation. + # The unified WarpGemm implementation does not yet support gfx1250. + if gfx_arch == "gfx1250": + arch_defines.append("-DUSE_NEW_UNIFIED_FRAMEWORK=0") # Family test on purpose: every gfx12xx part (gfx1200/gfx1201/gfx1250) uses OCP # FP8 encoding, so an exact-gfx1250 test here would be WRONG. This is the # opposite of the tile selector in codegen_common.rowcol_tensor_quant_default_tile(), @@ -603,6 +612,7 @@ def _compile_rowcolquant_kernel( compile_cmd = [hipcc, "-c", "-fPIC", "-O3", "-std=c++17", "-DCK_TILE_SINGLE_KERNEL_INCLUDE", "-w", f"--offload-arch={gfx_arch}", + f"-DCK_CMAKE_GPU_TARGET_IDS=0x{gfx_arch[3:]}", f"-DGFX_ARCH=\"{gfx_arch}\"", *arch_defines, "-include", str(hpp_path), @@ -680,7 +690,14 @@ def setup_multiple_rowcolquant_dispatchers( # Normalize the explicit branch too, not just detection: an explicitly passed # "gfx1250:xnack-" would otherwise flow into --offload-arch, -DGFX_ARCH and the # .so cache name. _detect_gpu_arch() already normalizes. - arch = normalize_gfx_arch(gfx_arch) if gfx_arch else _detect_gpu_arch() + arch = validate_rowcol_tensor_quant_gfx_arch(gfx_arch or _detect_gpu_arch()) + for cfg in configs: + config_arch = validate_rowcol_tensor_quant_gfx_arch(cfg.gfx_arch) + if config_arch != arch: + raise ValueError( + f"Config architecture {cfg.gfx_arch!r} does not match build target {arch!r}. " + "Create configs for the selected build target." + ) base_dir = output_dir or Path(tempfile.mkdtemp(prefix="rowcolquant_dispatcher_")) base_dir.mkdir(parents=True, exist_ok=True) @@ -771,7 +788,7 @@ def _default_config(dtype: str, gfx_arch: str) -> RowColQuantKernelConfig: # Normalize at this boundary too: default_fp8_config()/default_bf8_config() are # public entry points, so a caller-supplied "gfx1250:xnack-" must not be stored # on the config and observed by later consumers. - gfx_arch = normalize_gfx_arch(gfx_arch) + gfx_arch = validate_rowcol_tensor_quant_gfx_arch(gfx_arch) traits = ROWCOL_TENSOR_QUANT_DEFAULT_TRAITS return RowColQuantKernelConfig( dtype=dtype, diff --git a/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py b/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py index fc459ce41034..6d3e2d60057e 100644 --- a/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py +++ b/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py @@ -52,6 +52,7 @@ ROWCOL_TENSOR_QUANT_DEFAULT_TRAITS, rowcol_tensor_quant_default_tile, normalize_gfx_arch, + validate_rowcol_tensor_quant_gfx_arch, make_tensorquant_kernel_name, ) @@ -73,7 +74,8 @@ # than papered over with a runtime fallback. # 1 -> original export set # 2 -> added dispatcher_get_tile_n() / dispatcher_get_pad_n() -_SO_ABI = 2 +# 3 -> require regenerated headers with build-target validation +_SO_ABI = 3 # ============================================================================= @@ -204,7 +206,7 @@ class TensorQuantDispatcherLib: def __init__(self, so_path: Path): self.so_path = Path(so_path) - self._cleaned_up = False + self._cleaned_up = True if not self.so_path.exists(): raise FileNotFoundError(f"TensorQuant .so not found: {self.so_path}") self._lib = ctypes.CDLL(str(self.so_path)) @@ -212,6 +214,7 @@ def __init__(self, so_path: Path): rc = self._lib.dispatcher_initialize() if rc != 0: raise RuntimeError(f"dispatcher_initialize() returned {rc}") + self._cleaned_up = False def _setup(self): lib = self._lib @@ -246,16 +249,17 @@ def _setup(self): lib.dispatcher_get_kernel_count.restype = ctypes.c_int lib.dispatcher_get_kernel_count.argtypes = [] - # TileN / pad_n are compile-time properties of the single force-included - # kernel. They are read back rather than hardcoded so the N-divisibility - # diagnostics below stay correct if the default tile changes. No missing- - # symbol fallback: this .so is always compiled from the ctypes source that - # ships these exports, by this module. - lib.dispatcher_get_tile_n.restype = ctypes.c_int - lib.dispatcher_get_tile_n.argtypes = [] - - lib.dispatcher_get_pad_n.restype = ctypes.c_int - lib.dispatcher_get_pad_n.argtypes = [] + # Direct callers can supply libraries outside the versioned cache. + try: + lib.dispatcher_get_tile_n.restype = ctypes.c_int + lib.dispatcher_get_tile_n.argtypes = [] + lib.dispatcher_get_pad_n.restype = ctypes.c_int + lib.dispatcher_get_pad_n.argtypes = [] + except AttributeError as exc: + raise RuntimeError( + f"Incompatible quant bridge ABI in {self.so_path}: missing TileN/PadN " + "exports (ABI 2 or newer required). Rebuild the library." + ) from exc lib.dispatcher_cleanup.restype = None lib.dispatcher_cleanup.argtypes = [] @@ -530,6 +534,7 @@ def _generate_tensorquant_kernel( str(_CODEGEN_SCRIPT), "--output-dir", str(output_dir), "--config-json", config_json, + "--gfx-arch", validate_rowcol_tensor_quant_gfx_arch(config.gfx_arch), ] try: @@ -568,7 +573,7 @@ def _compile_tensorquant_kernel( # defines below and the --offload-arch/-DGFX_ARCH we hand to hipcc -- sees the # bare target. A caller-supplied "gfx1250:xnack-" must not reach the compiler # flags. - gfx_arch = normalize_gfx_arch(gfx_arch) + gfx_arch = validate_rowcol_tensor_quant_gfx_arch(gfx_arch) ck_include = _get_ck_include_dir() static_lib = _get_dispatcher_static_lib() @@ -576,6 +581,10 @@ def _compile_tensorquant_kernel( obj_path = so_path.with_suffix(".o") arch_defines = [] + # Match the top-level CK CMake policy in both host and device compilation. + # The unified WarpGemm implementation does not yet support gfx1250. + if gfx_arch == "gfx1250": + arch_defines.append("-DUSE_NEW_UNIFIED_FRAMEWORK=0") # Family test on purpose: every gfx12xx part (gfx1200/gfx1201/gfx1250) uses OCP # FP8 encoding, so an exact-gfx1250 test here would be WRONG. This is the # opposite of the tile selector in codegen_common.rowcol_tensor_quant_default_tile(), @@ -588,6 +597,7 @@ def _compile_tensorquant_kernel( compile_cmd = [hipcc, "-c", "-fPIC", "-O3", "-std=c++17", "-DCK_TILE_SINGLE_KERNEL_INCLUDE", "-w", f"--offload-arch={gfx_arch}", + f"-DCK_CMAKE_GPU_TARGET_IDS=0x{gfx_arch[3:]}", f"-DGFX_ARCH=\"{gfx_arch}\"", *arch_defines, "-include", str(hpp_path), @@ -665,7 +675,14 @@ def setup_multiple_tensorquant_dispatchers( # Normalize the explicit branch too, not just detection: an explicitly passed # "gfx1250:xnack-" would otherwise flow into --offload-arch, -DGFX_ARCH and the # .so cache name. _detect_gpu_arch() already normalizes. - arch = normalize_gfx_arch(gfx_arch) if gfx_arch else _detect_gpu_arch() + arch = validate_rowcol_tensor_quant_gfx_arch(gfx_arch or _detect_gpu_arch()) + for cfg in configs: + config_arch = validate_rowcol_tensor_quant_gfx_arch(cfg.gfx_arch) + if config_arch != arch: + raise ValueError( + f"Config architecture {cfg.gfx_arch!r} does not match build target {arch!r}. " + "Create configs for the selected build target." + ) base_dir = output_dir or Path(tempfile.mkdtemp(prefix="tensorquant_dispatcher_")) base_dir.mkdir(parents=True, exist_ok=True) @@ -756,7 +773,7 @@ def _default_config(dtype: str, gfx_arch: str) -> TensorQuantKernelConfig: # Normalize at this boundary too: default_fp8_config()/default_bf8_config() are # public entry points, so a caller-supplied "gfx1250:xnack-" must not be stored # on the config and observed by later consumers. - gfx_arch = normalize_gfx_arch(gfx_arch) + gfx_arch = validate_rowcol_tensor_quant_gfx_arch(gfx_arch) traits = ROWCOL_TENSOR_QUANT_DEFAULT_TRAITS return TensorQuantKernelConfig( dtype=dtype, diff --git a/projects/composablekernel/dispatcher/tests/CMakeLists.txt b/projects/composablekernel/dispatcher/tests/CMakeLists.txt index 3b9b5cce4779..10635a918ec8 100644 --- a/projects/composablekernel/dispatcher/tests/CMakeLists.txt +++ b/projects/composablekernel/dispatcher/tests/CMakeLists.txt @@ -221,12 +221,13 @@ endforeach() # silently collect zero tests and report success. # # The *_utils suites are pure Python (naming, config, codegen text) and run -# anywhere. The *_gpu_correctness suites arch-gate to gfx942/gfx950 and skip +# anywhere. The *_gpu_correctness suites arch-gate to gfx942/gfx950/gfx1250 and skip # cleanly without a GPU or hipcc, so they cost nothing on a CPU-only runner. # ============================================================================= foreach(_quant_test test_grouped_gemm_rowcolquant_utils - test_grouped_gemm_tensorquant_utils) + test_grouped_gemm_tensorquant_utils + test_quant_review_regressions) add_test( NAME dispatcher_${_quant_test} COMMAND ${Python3_EXECUTABLE} -m pytest ${CMAKE_CURRENT_SOURCE_DIR}/${_quant_test}.py -v diff --git a/projects/composablekernel/dispatcher/tests/test_codegen_common.py b/projects/composablekernel/dispatcher/tests/test_codegen_common.py index b464228ba013..ca041bfbd8e5 100644 --- a/projects/composablekernel/dispatcher/tests/test_codegen_common.py +++ b/projects/composablekernel/dispatcher/tests/test_codegen_common.py @@ -797,16 +797,13 @@ def test_the_error_names_the_value_and_the_alternatives(self): self.assertIn(arch, message) def test_the_set_matches_the_bridge_runtime(self): - # One advertised support set per bridge. gemm_utils is the runtime entry - # point; if the two drift, --gfx-arch accepts a target the runner refuses, - # or refuses one it accepts. - sys.path.insert(0, str(DISPATCHER_DIR / "python")) - import gemm_utils - self.assertEqual( - sorted(ROWCOL_TENSOR_QUANT_SUPPORTED_ARCHES), - sorted(gemm_utils._SUPPORTED_ARCHES), + ROWCOL_TENSOR_QUANT_SUPPORTED_ARCHES, ("gfx942", "gfx950", "gfx1250") ) + for arch in ("gfx90a", "gfx90a:xnack-"): + with self.assertRaises(ValueError): + validate_rowcol_tensor_quant_gfx_arch(arch) + if __name__ == "__main__": diff --git a/projects/composablekernel/dispatcher/tests/test_grouped_gemm_rowcolquant_utils.py b/projects/composablekernel/dispatcher/tests/test_grouped_gemm_rowcolquant_utils.py index 74c6ab3f0f77..d4e861a1a3d8 100644 --- a/projects/composablekernel/dispatcher/tests/test_grouped_gemm_rowcolquant_utils.py +++ b/projects/composablekernel/dispatcher/tests/test_grouped_gemm_rowcolquant_utils.py @@ -383,14 +383,17 @@ def fake_run(cmd, *a, **kw): assert ok, "compile helper should report success when hipcc succeeds" return captured[0] - @pytest.mark.parametrize("arch", ["gfx1200", "gfx1201", "gfx1250", "gfx1250:xnack-"]) - def test_all_gfx12_parts_get_ocp_fp8(self, monkeypatch, tmp_path, arch): + @pytest.mark.parametrize("arch", ["gfx1250", "gfx1250:xnack-"]) + def test_supported_gfx12_targets_get_ocp_fp8(self, monkeypatch, tmp_path, arch): argv = self._compile_argv(monkeypatch, tmp_path, arch) + assert "-DUSE_NEW_UNIFIED_FRAMEWORK=0" in argv + assert "-DCK_CMAKE_GPU_TARGET_IDS=0x1250" in argv assert "-DCK_USE_OCP_FP8" in argv assert "-DCK_TILE_USE_OCP_FP8" in argv def test_gfx950_gets_ocp_fp8_and_mx(self, monkeypatch, tmp_path): argv = self._compile_argv(monkeypatch, tmp_path, "gfx950") + assert "-DCK_CMAKE_GPU_TARGET_IDS=0x950" in argv assert "-DCK_USE_OCP_FP8" in argv assert "-DCK_USE_NATIVE_MX_SUPPORT" in argv @@ -467,13 +470,14 @@ class TestSoCacheAbiKey: def test_abi_is_versioned(self): assert isinstance(UTILS._SO_ABI, int) and UTILS._SO_ABI >= 2 - def test_pre_abi_artifact_is_not_reused(self, monkeypatch, tmp_path): + @pytest.mark.parametrize("old_suffix", ["", "_abi2"]) + def test_pre_abi_artifact_is_not_reused(self, monkeypatch, tmp_path, old_suffix): cfg = default_fp8_config(gfx_arch="gfx950") so_dir = tmp_path / "libs" so_dir.mkdir(parents=True) # Exactly the name the old code would have produced and reused. - stale = so_dir / f"lib{cfg.name}_gfx950.so" + stale = so_dir / f"lib{cfg.name}_gfx950{old_suffix}.so" stale.write_bytes(b"stale pre-ABI artifact") compiled = [] diff --git a/projects/composablekernel/dispatcher/tests/test_grouped_gemm_tensorquant_utils.py b/projects/composablekernel/dispatcher/tests/test_grouped_gemm_tensorquant_utils.py index 289ad64d6dee..359ea2a8c683 100644 --- a/projects/composablekernel/dispatcher/tests/test_grouped_gemm_tensorquant_utils.py +++ b/projects/composablekernel/dispatcher/tests/test_grouped_gemm_tensorquant_utils.py @@ -390,14 +390,17 @@ def fake_run(cmd, *a, **kw): assert ok, "compile helper should report success when hipcc succeeds" return captured[0] - @pytest.mark.parametrize("arch", ["gfx1200", "gfx1201", "gfx1250", "gfx1250:xnack-"]) - def test_all_gfx12_parts_get_ocp_fp8(self, monkeypatch, tmp_path, arch): + @pytest.mark.parametrize("arch", ["gfx1250", "gfx1250:xnack-"]) + def test_supported_gfx12_targets_get_ocp_fp8(self, monkeypatch, tmp_path, arch): argv = self._compile_argv(monkeypatch, tmp_path, arch) + assert "-DUSE_NEW_UNIFIED_FRAMEWORK=0" in argv + assert "-DCK_CMAKE_GPU_TARGET_IDS=0x1250" in argv assert "-DCK_USE_OCP_FP8" in argv assert "-DCK_TILE_USE_OCP_FP8" in argv def test_gfx950_gets_ocp_fp8_and_mx(self, monkeypatch, tmp_path): argv = self._compile_argv(monkeypatch, tmp_path, "gfx950") + assert "-DCK_CMAKE_GPU_TARGET_IDS=0x950" in argv assert "-DCK_USE_OCP_FP8" in argv assert "-DCK_USE_NATIVE_MX_SUPPORT" in argv @@ -474,13 +477,14 @@ class TestSoCacheAbiKey: def test_abi_is_versioned(self): assert isinstance(UTILS._SO_ABI, int) and UTILS._SO_ABI >= 2 - def test_pre_abi_artifact_is_not_reused(self, monkeypatch, tmp_path): + @pytest.mark.parametrize("old_suffix", ["", "_abi2"]) + def test_pre_abi_artifact_is_not_reused(self, monkeypatch, tmp_path, old_suffix): cfg = default_fp8_config(gfx_arch="gfx950") so_dir = tmp_path / "libs" so_dir.mkdir(parents=True) # Exactly the name the old code would have produced and reused. - stale = so_dir / f"lib{cfg.name}_gfx950.so" + stale = so_dir / f"lib{cfg.name}_gfx950{old_suffix}.so" stale.write_bytes(b"stale pre-ABI artifact") compiled = [] diff --git a/projects/composablekernel/dispatcher/tests/test_quant_review_regressions.py b/projects/composablekernel/dispatcher/tests/test_quant_review_regressions.py new file mode 100644 index 000000000000..bf309e858267 --- /dev/null +++ b/projects/composablekernel/dispatcher/tests/test_quant_review_regressions.py @@ -0,0 +1,178 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +"""Host regressions for quant build targets, direct loading, and C++ guards.""" + +import ctypes +import importlib +import json +from pathlib import Path +import shutil +import subprocess +import sys +from types import SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path[:0] = [str(ROOT / "python"), str(ROOT / "codegen")] +OPS = ("rowcolquant", "tensorquant") +CXX = shutil.which("c++") + + +@pytest.mark.parametrize("op", OPS) +@pytest.mark.parametrize("config_arch,build_arch", [("gfx1250", "gfx942"), ("gfx942", "gfx1250")]) +@pytest.mark.parametrize("explicit", [False, True]) +def test_config_build_mismatch_fails_before_codegen(op, config_arch, build_arch, explicit, monkeypatch, tmp_path): + module = importlib.import_module(f"grouped_gemm_{op}_utils") + monkeypatch.setattr(module, "_detect_gpu_arch", lambda: build_arch) + monkeypatch.setattr(module, f"_generate_{op}_kernel", lambda *a: pytest.fail("codegen must not run")) + setup = getattr(module, f"setup_multiple_{op}_dispatchers") + with pytest.raises(ValueError, match="does not match build target"): + setup([module.default_fp8_config(config_arch)], output_dir=tmp_path / "output", + gfx_arch=build_arch if explicit else None) + assert not (tmp_path / "output").exists() + + +@pytest.mark.parametrize("op", OPS) +def test_matching_suffix_config_generates_target_metadata(op, tmp_path): + module = importlib.import_module(f"grouped_gemm_{op}_utils") + config = module.default_fp8_config("gfx942:sramecc+:xnack-") + config.gfx_arch = "gfx942:xnack-" + header = getattr(module, f"_generate_{op}_kernel")(config, tmp_path) + assert header is not None + assert 'GfxArch = "gfx942"' in header.read_text() + + +@pytest.mark.parametrize("op", OPS) +@pytest.mark.parametrize("arch", ["gfx90a", "gfx90a:xnack-", "gfx1200", "gfx1201"]) +def test_unsupported_quant_targets_rejected_at_entry(op, arch, tmp_path): + module = importlib.import_module(f"grouped_gemm_{op}_utils") + with pytest.raises(ValueError, match="Unsupported GPU architecture"): + module.default_fp8_config(arch) + result = subprocess.run([sys.executable, str(ROOT / f"codegen/unified_grouped_gemm_{op}_codegen.py"), + "--gfx-arch", arch, "--output-dir", str(tmp_path)], capture_output=True, text=True) + assert result.returncode != 0 + assert "Unsupported GPU architecture" in result.stderr + assert not list(tmp_path.glob("*.hpp")) + + +@pytest.mark.parametrize("op,cls", [("rowcolquant", "RowColQuant"), ("tensorquant", "TensorQuant")]) +@pytest.mark.parametrize("missing", ["dispatcher_get_tile_n", "dispatcher_get_pad_n"]) +def test_direct_runner_reports_old_abi(op, cls, missing, monkeypatch, tmp_path): + module = importlib.import_module(f"grouped_gemm_{op}_utils") + lib = SimpleNamespace() + lifecycle_calls = [] + for name in ("dispatcher_initialize", "dispatcher_run_gemm", "dispatcher_get_kernel_name", + "dispatcher_get_kernel_count", "dispatcher_get_tile_n", "dispatcher_get_pad_n", + "dispatcher_cleanup"): + if name != missing: + setattr(lib, name, lambda *args: 0) + lib.dispatcher_initialize = lambda: lifecycle_calls.append("initialize") + lib.dispatcher_cleanup = lambda: lifecycle_calls.append("cleanup") + path = tmp_path / "old.so" + path.touch() + monkeypatch.setattr(module.ctypes, "CDLL", lambda *a: lib) + with pytest.raises(RuntimeError, match="Incompatible quant bridge ABI.*Rebuild"): + getattr(module, cls + "GpuGemmRunner")(path) + assert lifecycle_calls == [] + + +def compile_cpp(tmp_path, source, shared=False): + if not CXX: + pytest.skip("requires C++ compiler") + cpp = tmp_path / "probe.cpp" + cpp.write_text(source) + output = tmp_path / ("probe.so" if shared else "probe.o") + args = [CXX, "-std=c++17", str(cpp), "-o", str(output)] + args += ["-shared", "-fPIC"] if shared else ["-c"] + return subprocess.run(args, capture_output=True, text=True), output + + +@pytest.mark.parametrize("op", OPS) +@pytest.mark.parametrize("generated,compiled,passes", [ + ("gfx1250", "gfx942", False), ("gfx942", "gfx1250", False), + ("gfx1250", "gfx1250", True), ("gfx942", "gfx942", True), ("gfx950", "gfx950", True), +]) +def test_production_cpp_rejects_header_target_mismatch(op, generated, compiled, passes, tmp_path): + codegen = importlib.import_module(f"unified_grouped_gemm_{op}_codegen") + header = codegen.generate_kernels(tmp_path / "headers", gfx_arch=generated, parallel=False)[0].read_text() + # Compile the actual emitted constants and actual bridge guard block without + # requiring HIP. Kernel implementation and GPU execution are tested separately. + constants = header[header.index(" static constexpr const char* GfxArch"):header.index(" // Informational only:")] + constants = constants.replace("ck_tile::index_t", "int") + bridge = (ROOT / f"bindings/ctypes/grouped_gemm_{op}_ctypes_lib.cpp").read_text() + guards = bridge[bridge.index("static constexpr bool ct_starts_with"):bridge.index('extern "C" {')] + source = f'#define GFX_ARCH "{compiled}"\nstruct SelectedKernel {{\n{constants}\n}};\n{guards}' + result, _ = compile_cpp(tmp_path, source) + assert (result.returncode == 0) == passes, result.stderr + if not passes: + assert "Generated kernel architecture does not match" in result.stderr + + +@pytest.mark.parametrize("arch,pad_n,M,N,expected", [ + ("gfx1250", False, 128, 128, 0), + ("gfx1250", False, 126, 128, -1), + ("gfx1250", False, 128, 100, -1), + ("gfx942", False, 126, 128, 0), + ("gfx1250", True, 128, 100, 0), +]) +def test_rowcolquant_actual_shape_guards(arch, pad_n, M, N, expected, tmp_path): + bridge = (ROOT / "bindings/ctypes/grouped_gemm_rowcolquant_ctypes_lib.cpp").read_text() + # Execute the production host-side prefix of dispatcher_run_gemm. Stop before + # allocation/launch; valid inputs return a sentinel success in this harness. + prefix = bridge[bridge.index("int dispatcher_run_gemm("):bridge.index(" // Only packed (contiguous) layouts")] + source = f'''#include +#include +#include +#define GFX_ARCH "{arch}" +std::atomic g_ref_count{{1}}; +constexpr bool kCompiledForGfx12 = {str(arch == "gfx1250").lower()}; +struct SelectedKernel {{ static constexpr bool kPadN = {str(pad_n).lower()}; static constexpr int TileN = 64; }}; +extern "C" {{ +{prefix} +return 0; +}} +}} +''' + result, output = compile_cpp(tmp_path, source, shared=True) + assert result.returncode == 0, result.stderr + lib = ctypes.CDLL(str(output)) + run = lib.dispatcher_run_gemm + run.argtypes = [ctypes.c_void_p] * 5 + [ctypes.c_int64] * 10 + [ctypes.c_int, ctypes.POINTER(ctypes.c_float)] + run.restype = ctypes.c_int + elapsed = ctypes.c_float() + assert run(*([1] * 5), M, N, 192, 192, 192, 1, 1, N, M, N, 1, ctypes.byref(elapsed)) == expected + + +@pytest.mark.parametrize("explicit", [True, False]) +@pytest.mark.parametrize("arch", ["gfx942:sramecc+:xnack-", "gfx1250"]) +def test_cmake_normalizes_explicit_and_inferred_arches(tmp_path, explicit, arch): + if not shutil.which("cmake"): + pytest.skip("requires CMake") + source = tmp_path / "source" + source.mkdir() + build = tmp_path / "build" + headers = build / "generated_kernels" + headers.mkdir(parents=True) + for op in OPS: + (headers / f"grouped_gemm_{op}_test.hpp").touch() + overrides = '\n'.join(f'set(CK_TILE_{op.upper()}_GFX_ARCH "{arch}")' for op in OPS) if explicit else '' + (source / "CMakeLists.txt").write_text(f'''cmake_minimum_required(VERSION 3.16) +project(quant_cmake_probe LANGUAGES CXX) +add_library(hip::device INTERFACE IMPORTED) +set(CMAKE_HIP_ARCHITECTURES "{arch}") +{overrides} +add_subdirectory("{ROOT / 'bindings/ctypes'}" ctypes) +''') + result = subprocess.run(["cmake", "-S", str(source), "-B", str(build), "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON"], capture_output=True, text=True) + assert result.returncode == 0, result.stdout + result.stderr + commands = json.loads((build / "compile_commands.json").read_text()) + for op in OPS: + command = next(c["command"] for c in commands if c["file"].endswith(f"grouped_gemm_{op}_ctypes_lib.cpp")) + base = arch.split(":")[0] + assert f'GFX_ARCH=\\"{base}\\"' in command + assert f'GFX_ARCH=\\"{base}:' not in command + assert f'-DCK_CMAKE_GPU_TARGET_IDS=0x{base[3:]}' in command + if base == "gfx1250": + assert "-DUSE_NEW_UNIFIED_FRAMEWORK=0" in command From 4aefed20760c144565b117842f199dee61c65321 Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk <3836908+ozturkosu@users.noreply.github.com> Date: Fri, 18 Sep 2026 02:36:38 -0400 Subject: [PATCH 21/24] fix(ck): align standalone quant CMake architecture features Apply target-scoped feature definitions matching the Python quant builds for gfx942, gfx950 and gfx1250. Preserve the gfx1250 legacy framework override and validate explicit/inferred, bare/suffixed targets against arch_feature_defines(). Regression: eight parity cases fail before the fix. Afterward 97 tests and 45 subtests plus 9/9 selected CTest suites pass. Fresh standalone CMake builds on MI400 pass all six RowColQuant/TensorQuant reference cases. --- .../dispatcher/bindings/ctypes/CMakeLists.txt | 28 +++++++++++++++---- .../tests/test_quant_review_regressions.py | 18 ++++++++++-- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/CMakeLists.txt b/projects/composablekernel/dispatcher/bindings/ctypes/CMakeLists.txt index dd25d4f539ff..cc64b259749f 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/CMakeLists.txt +++ b/projects/composablekernel/dispatcher/bindings/ctypes/CMakeLists.txt @@ -48,6 +48,24 @@ function(add_ctypes_library TARGET_NAME SOURCE_FILE) ) endfunction() +# Feature definitions for the two single-target grouped quant bridges below. +# Keep these aligned with dispatcher_common.arch_feature_defines(): standalone +# CMake builds do not inherit the top-level CK definitions, and host/device FP8 +# encoding must agree. test_quant_review_regressions checks CMake/JIT parity. +function(set_grouped_quant_arch_features TARGET_NAME GFX_ARCH) + if(GFX_ARCH STREQUAL "gfx950" OR GFX_ARCH STREQUAL "gfx1250") + target_compile_definitions(${TARGET_NAME} PRIVATE + CK_USE_OCP_FP8 CK_TILE_USE_OCP_FP8 CK_USE_NATIVE_MX_SUPPORT) + endif() + if(GFX_ARCH STREQUAL "gfx950") + target_compile_definitions(${TARGET_NAME} PRIVATE CK_GFX950_SUPPORT) + elseif(GFX_ARCH STREQUAL "gfx1250") + target_compile_definitions(${TARGET_NAME} PRIVATE + CK_TILE_USE_WMMA=1 CK_GFX12_SUPPORT CK_USE_GFX1250 CK_GFX1250_SUPPORT + USE_NEW_UNIFIED_FRAMEWORK=0) + endif() +endfunction() + # Helper function for block-scale quant ctypes targets. # # Usage: @@ -363,9 +381,8 @@ if(ROWCOLQUANT_GEMM_KERNEL_HEADERS) string(REGEX REPLACE "^gfx" "0x" _rowcolquant_target_id "${CK_TILE_ROWCOLQUANT_GFX_ARCH}") target_compile_options(dispatcher_grouped_gemm_rowcolquant_lib PRIVATE -UCK_CMAKE_GPU_TARGET_IDS "-DCK_CMAKE_GPU_TARGET_IDS=${_rowcolquant_target_id}") - if(CK_TILE_ROWCOLQUANT_GFX_ARCH STREQUAL "gfx1250") - target_compile_definitions(dispatcher_grouped_gemm_rowcolquant_lib PRIVATE USE_NEW_UNIFIED_FRAMEWORK=0) - endif() + set_grouped_quant_arch_features(dispatcher_grouped_gemm_rowcolquant_lib + "${CK_TILE_ROWCOLQUANT_GFX_ARCH}") target_compile_definitions(dispatcher_grouped_gemm_rowcolquant_lib PRIVATE CK_TILE_SINGLE_KERNEL_INCLUDE @@ -423,9 +440,8 @@ if(TENSORQUANT_GEMM_KERNEL_HEADERS) string(REGEX REPLACE "^gfx" "0x" _tensorquant_target_id "${CK_TILE_TENSORQUANT_GFX_ARCH}") target_compile_options(dispatcher_grouped_gemm_tensorquant_lib PRIVATE -UCK_CMAKE_GPU_TARGET_IDS "-DCK_CMAKE_GPU_TARGET_IDS=${_tensorquant_target_id}") - if(CK_TILE_TENSORQUANT_GFX_ARCH STREQUAL "gfx1250") - target_compile_definitions(dispatcher_grouped_gemm_tensorquant_lib PRIVATE USE_NEW_UNIFIED_FRAMEWORK=0) - endif() + set_grouped_quant_arch_features(dispatcher_grouped_gemm_tensorquant_lib + "${CK_TILE_TENSORQUANT_GFX_ARCH}") target_compile_definitions(dispatcher_grouped_gemm_tensorquant_lib PRIVATE CK_TILE_SINGLE_KERNEL_INCLUDE diff --git a/projects/composablekernel/dispatcher/tests/test_quant_review_regressions.py b/projects/composablekernel/dispatcher/tests/test_quant_review_regressions.py index bf309e858267..f65ea96caad0 100644 --- a/projects/composablekernel/dispatcher/tests/test_quant_review_regressions.py +++ b/projects/composablekernel/dispatcher/tests/test_quant_review_regressions.py @@ -7,6 +7,7 @@ import importlib import json from pathlib import Path +import shlex import shutil import subprocess import sys @@ -146,7 +147,11 @@ def test_rowcolquant_actual_shape_guards(arch, pad_n, M, N, expected, tmp_path): @pytest.mark.parametrize("explicit", [True, False]) -@pytest.mark.parametrize("arch", ["gfx942:sramecc+:xnack-", "gfx1250"]) +@pytest.mark.parametrize("arch", [ + "gfx942", "gfx942:sramecc+:xnack-", + "gfx950", "gfx950:sramecc+:xnack-", + "gfx1250", "gfx1250:xnack-", +]) def test_cmake_normalizes_explicit_and_inferred_arches(tmp_path, explicit, arch): if not shutil.which("cmake"): pytest.skip("requires CMake") @@ -168,11 +173,20 @@ def test_cmake_normalizes_explicit_and_inferred_arches(tmp_path, explicit, arch) result = subprocess.run(["cmake", "-S", str(source), "-B", str(build), "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON"], capture_output=True, text=True) assert result.returncode == 0, result.stdout + result.stderr commands = json.loads((build / "compile_commands.json").read_text()) + from dispatcher_common import arch_feature_defines + + # Compare the standalone CMake command with the working Python compiler + # path, including absent flags so gfx1250/OCP settings cannot leak to gfx942. + feature_flags = set().union(*(set(arch_feature_defines(a)) for a in ("gfx942", "gfx950", "gfx1250"))) + feature_flags.add("-DUSE_NEW_UNIFIED_FRAMEWORK=0") for op in OPS: command = next(c["command"] for c in commands if c["file"].endswith(f"grouped_gemm_{op}_ctypes_lib.cpp")) base = arch.split(":")[0] assert f'GFX_ARCH=\\"{base}\\"' in command assert f'GFX_ARCH=\\"{base}:' not in command assert f'-DCK_CMAKE_GPU_TARGET_IDS=0x{base[3:]}' in command + expected = set(arch_feature_defines(base)) if base == "gfx1250": - assert "-DUSE_NEW_UNIFIED_FRAMEWORK=0" in command + expected.add("-DUSE_NEW_UNIFIED_FRAMEWORK=0") + actual = set(shlex.split(command)) & feature_flags + assert actual == expected, f"{op} {arch}: CMake/JIT feature mismatch: {actual ^ expected}" From 031ef075242ec00e7c8eb3bfbc9415013d602605 Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk <3836908+ozturkosu@users.noreply.github.com> Date: Fri, 18 Sep 2026 02:41:10 -0400 Subject: [PATCH 22/24] fix(ck): scope quant warp restrictions to grouped bridges Move the gfx1250 warp-count and warp_k gates to the operator routing boundary so plain gemm_rowcolquant retains shared validation without grouped codegen restrictions. Keep common fragment checks for all three operators. Add routing regressions for FP8/BF8 and bare/suffixed gfx1250 targets: eight plain RowColQuant cases reproduce the bug before the fix. Afterward 113 tests and 423 subtests plus all nine selected CTest suites pass. --- ...test_gemm_validation_arch_normalization.py | 39 ++++++++++++++++++ .../ops/gemm/gemm_validation_utils.py | 40 +++++++------------ 2 files changed, 54 insertions(+), 25 deletions(-) diff --git a/projects/composablekernel/dispatcher/tests/test_gemm_validation_arch_normalization.py b/projects/composablekernel/dispatcher/tests/test_gemm_validation_arch_normalization.py index b9167c7fcbf4..cdc737d4d1ee 100644 --- a/projects/composablekernel/dispatcher/tests/test_gemm_validation_arch_normalization.py +++ b/projects/composablekernel/dispatcher/tests/test_gemm_validation_arch_normalization.py @@ -211,6 +211,45 @@ def test_gfx1201_suffix_is_not_swept_up_by_the_gfx1250_exception(self): self.assertTrue(all(self._decisions(_SUFFIXES["gfx1201"]).values())) +@unittest.skipIf(gvu is None, "tile_engine tree not present") +class TestGroupedQuantWarpRestrictionScope(unittest.TestCase): + """Exercise operator routing, not just the shared quant validator.""" + + PREFIXES = ("gemm_rowcolquant", "grouped_gemm_rowcolquant", "grouped_gemm_tensorquant") + + @staticmethod + def config(arch, dtype, warp): + return dict( + tile_m=128, tile_n=128, tile_k=256, + warp_m=warp[0], warp_n=warp[1], warp_k=warp[2], + warp_tile_m=16, warp_tile_n=16, warp_tile_k=128, + a_datatype=dtype, b_datatype=dtype, c_datatype="fp16", + pipeline="compv3", layout="rcr", gpu_target=arch, + ) + + def test_grouped_limits_do_not_filter_plain_rowcolquant(self): + for arch, dtype, prefix, warp in itertools.product( + ("gfx1250", "gfx1250:xnack-"), ("fp8", "bf8"), self.PREFIXES, + ((4, 2, 1), (1, 2, 2), (1, 4, 1)), + ): + with self.subTest(arch=arch, dtype=dtype, prefix=prefix, warp=warp): + # Eight warps and warp_k=2 are grouped-bridge restrictions. + # Four warps with warp_k=1 remain valid for all three operators. + expected = prefix == "gemm_rowcolquant" or warp == (1, 4, 1) + self.assertEqual(gvu.is_tile_config_valid( + **self.config(arch, dtype, warp), kernel_name_prefix=prefix, + ), expected) + + def test_shared_fragment_validation_still_applies_to_all_quant_operators(self): + for arch, dtype, prefix in itertools.product( + ("gfx1250", "gfx1250:xnack-"), ("fp8", "bf8"), self.PREFIXES, + ): + with self.subTest(arch=arch, dtype=dtype, prefix=prefix): + cfg = self.config(arch, dtype, (1, 4, 1)) + cfg.update(warp_tile_m=32, warp_tile_n=32) + self.assertFalse(gvu.is_tile_config_valid(**cfg, kernel_name_prefix=prefix)) + + if __name__ == "__main__": logging.disable(logging.CRITICAL) sys.exit(unittest.main()) diff --git a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py index 1ebf8ed806be..4b5a4330215e 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py +++ b/projects/composablekernel/tile_engine/ops/gemm/gemm_validation_utils.py @@ -769,6 +769,21 @@ def is_tile_config_valid( return False # Additional operator-specific validation (runs after pipeline validation) + # These limits come from the two grouped quant bridges' code generation, + # not from gfx1250 hardware or the shared RowColQuant fragment rules. Keep + # them at the routing boundary so plain gemm_rowcolquant is not filtered. + if kernel_name_prefix in ("grouped_gemm_rowcolquant", "grouped_gemm_tensorquant") and ( + _base_gfx_arch(gpu_target) == "gfx1250" + ): + if warp_m * warp_n * warp_k > 4: + logging.debug( + "gfx1250 grouped quant bridges require at most four warps per block" + ) + return False + if warp_k > 1: + logging.debug("gfx1250 grouped quant bridges require warp_k=1") + return False + if ( kernel_name_prefix == "gemm_rowcolquant" or kernel_name_prefix == "grouped_gemm_rowcolquant" @@ -1522,31 +1537,6 @@ def validate_gemm_rowcol_tensor_quant( gpu_target: str, ) -> Tuple[bool, str]: """Validate RowColQuant / TensorQuant GEMM-specific constraints.""" - # gfx1250 warp-map restriction, deliberately scoped to THIS operator rather than - # placed in WARP_SUPPORTED_COMBINATIONS. That table is shared by seven instance - # builders, and plain GEMM's default gfx1250 WMMA config is an 8-warp 4x2x1 map - # that issue #11161 / PR #11175 fixed and verified correct on device -- removing - # it from the shared table would delete another operator's working default. - # - # These two grouped quant bridges are different: their codegen does not emit a - # loadable entry for >4-warp blocks on gfx1250, so those kernels abort at launch - # with "cannot find symbol" -- 3,352 of the 4,320 8-warp rows in a 14,208-row - # sweep, and 2,168 of those even when paired with an otherwise legal tile. That - # is a property of this bridge's code generation, not of the hardware, so the - # rule belongs here. (Both figures come from the same archive; an earlier draft - # of this comment quoted 3,220, which is from a different, smaller sweep.) - if _base_gfx_arch(gpu_target) == "gfx1250": - if warp_m * warp_n * warp_k > 4: - return False, ( - f"On gfx1250 these grouped quant bridges do not emit a loadable " - f"kernel entry for blocks of more than four warps; got " - f"{warp_m}x{warp_n}x{warp_k} ({warp_m * warp_n * warp_k} warps)" - ) - if warp_k > 1: - return False, ( - f"On gfx1250 warp_k must be 1 for these bridges; got warp_k={warp_k}" - ) - whole_workgroup_cover_valid, whole_workgroup_cover_error = ( validate_whole_wg_cover_configuration( tile_m, From aff4c2fecbe6ef8d88748b0d5355b53b7a48202f Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk <3836908+ozturkosu@users.noreply.github.com> Date: Fri, 18 Sep 2026 03:44:24 -0400 Subject: [PATCH 23/24] fix(ck): isolate grouped quant architecture features --- .../dispatcher/bindings/ctypes/CMakeLists.txt | 31 ++++++++++---- .../tests/test_quant_review_regressions.py | 41 +++++++++++++++---- 2 files changed, 56 insertions(+), 16 deletions(-) diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/CMakeLists.txt b/projects/composablekernel/dispatcher/bindings/ctypes/CMakeLists.txt index cc64b259749f..624afaee9f38 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/CMakeLists.txt +++ b/projects/composablekernel/dispatcher/bindings/ctypes/CMakeLists.txt @@ -49,20 +49,33 @@ function(add_ctypes_library TARGET_NAME SOURCE_FILE) endfunction() # Feature definitions for the two single-target grouped quant bridges below. -# Keep these aligned with dispatcher_common.arch_feature_defines(): standalone -# CMake builds do not inherit the top-level CK definitions, and host/device FP8 -# encoding must agree. test_quant_review_regressions checks CMake/JIT parity. +# Keep these aligned with dispatcher_common.arch_feature_defines(). Parent CK +# builds enable features directory-wide for the union of their GPU targets. +# Clear those definitions for these bridges before applying the selected arch, +# so host/device FP8 encoding and the backend match the standalone JIT path. function(set_grouped_quant_arch_features TARGET_NAME GFX_ARCH) + # Compile options follow directory/target definitions on the command line. + # Keep both the undefines and replacement defines here to preserve ordering + # without changing the inherited definitions of any sibling target. + foreach(_feature IN ITEMS + CK_USE_XDL CK_USE_GFX94 CK_USE_GFX950 CK_GFX1030_SUPPORT + CK_USE_WMMA CK_TILE_USE_WMMA CK_USE_WMMA_FP8 + CK_USE_OCP_FP8 CK_TILE_USE_OCP_FP8 CK_USE_FNUZ_FP8 + CK_USE_NATIVE_MX_SUPPORT CK_GFX950_SUPPORT + CK_USE_GFX1250 CK_GFX1250_SUPPORT CK_GFX12_SUPPORT + USE_NEW_UNIFIED_FRAMEWORK) + target_compile_options(${TARGET_NAME} PRIVATE "-U${_feature}") + endforeach() if(GFX_ARCH STREQUAL "gfx950" OR GFX_ARCH STREQUAL "gfx1250") - target_compile_definitions(${TARGET_NAME} PRIVATE - CK_USE_OCP_FP8 CK_TILE_USE_OCP_FP8 CK_USE_NATIVE_MX_SUPPORT) + target_compile_options(${TARGET_NAME} PRIVATE + -DCK_USE_OCP_FP8 -DCK_TILE_USE_OCP_FP8 -DCK_USE_NATIVE_MX_SUPPORT) endif() if(GFX_ARCH STREQUAL "gfx950") - target_compile_definitions(${TARGET_NAME} PRIVATE CK_GFX950_SUPPORT) + target_compile_options(${TARGET_NAME} PRIVATE -DCK_GFX950_SUPPORT) elseif(GFX_ARCH STREQUAL "gfx1250") - target_compile_definitions(${TARGET_NAME} PRIVATE - CK_TILE_USE_WMMA=1 CK_GFX12_SUPPORT CK_USE_GFX1250 CK_GFX1250_SUPPORT - USE_NEW_UNIFIED_FRAMEWORK=0) + target_compile_options(${TARGET_NAME} PRIVATE + -DCK_TILE_USE_WMMA=1 -DCK_GFX12_SUPPORT -DCK_USE_GFX1250 -DCK_GFX1250_SUPPORT + -DUSE_NEW_UNIFIED_FRAMEWORK=0) endif() endfunction() diff --git a/projects/composablekernel/dispatcher/tests/test_quant_review_regressions.py b/projects/composablekernel/dispatcher/tests/test_quant_review_regressions.py index f65ea96caad0..b6f29fb33721 100644 --- a/projects/composablekernel/dispatcher/tests/test_quant_review_regressions.py +++ b/projects/composablekernel/dispatcher/tests/test_quant_review_regressions.py @@ -146,13 +146,14 @@ def test_rowcolquant_actual_shape_guards(arch, pad_n, M, N, expected, tmp_path): assert run(*([1] * 5), M, N, 192, 192, 192, 1, 1, N, M, N, 1, ctypes.byref(elapsed)) == expected +@pytest.mark.parametrize("inherited_arches", [None, "gfx942;gfx1250", "gfx950;gfx942", "gfx1250;gfx950"]) @pytest.mark.parametrize("explicit", [True, False]) @pytest.mark.parametrize("arch", [ "gfx942", "gfx942:sramecc+:xnack-", "gfx950", "gfx950:sramecc+:xnack-", "gfx1250", "gfx1250:xnack-", ]) -def test_cmake_normalizes_explicit_and_inferred_arches(tmp_path, explicit, arch): +def test_cmake_normalizes_explicit_and_inferred_arches(tmp_path, explicit, arch, inherited_arches): if not shutil.which("cmake"): pytest.skip("requires CMake") source = tmp_path / "source" @@ -163,10 +164,19 @@ def test_cmake_normalizes_explicit_and_inferred_arches(tmp_path, explicit, arch) for op in OPS: (headers / f"grouped_gemm_{op}_test.hpp").touch() overrides = '\n'.join(f'set(CK_TILE_{op.upper()}_GFX_ARCH "{arch}")' for op in OPS) if explicit else '' + # Execute the real parent feature-definition block, including its defaults. + parent_features = "" + if inherited_arches: + parent_cmake = (ROOT.parent / "CMakeLists.txt").read_text() + start = parent_cmake.index('if (SUPPORTED_GPU_TARGETS MATCHES "gfx9|gfx11|gfx12"') + end = parent_cmake.index('if ((SUPPORTED_GPU_TARGETS MATCHES "gfx942"', start) + parent_features = f'set(SUPPORTED_GPU_TARGETS "{inherited_arches}")\n' + parent_cmake[start:end] + configured_arches = f"{arch};{inherited_arches}" if inherited_arches else arch (source / "CMakeLists.txt").write_text(f'''cmake_minimum_required(VERSION 3.16) project(quant_cmake_probe LANGUAGES CXX) add_library(hip::device INTERFACE IMPORTED) -set(CMAKE_HIP_ARCHITECTURES "{arch}") +{parent_features} +set(CMAKE_HIP_ARCHITECTURES "{configured_arches}") {overrides} add_subdirectory("{ROOT / 'bindings/ctypes'}" ctypes) ''') @@ -175,10 +185,24 @@ def test_cmake_normalizes_explicit_and_inferred_arches(tmp_path, explicit, arch) commands = json.loads((build / "compile_commands.json").read_text()) from dispatcher_common import arch_feature_defines - # Compare the standalone CMake command with the working Python compiler - # path, including absent flags so gfx1250/OCP settings cannot leak to gfx942. + # Compare effective macros, preserving the generated command's -D/-U order. + # Checking only flag presence misses inherited definitions and overrides. feature_flags = set().union(*(set(arch_feature_defines(a)) for a in ("gfx942", "gfx950", "gfx1250"))) - feature_flags.add("-DUSE_NEW_UNIFIED_FRAMEWORK=0") + feature_flags.update({"-DUSE_NEW_UNIFIED_FRAMEWORK=0", "-DCK_USE_FNUZ_FP8", + "-DCK_USE_XDL", "-DCK_USE_GFX94", "-DCK_USE_GFX950", + "-DCK_USE_WMMA", "-DCK_USE_WMMA_FP8", "-DCK_GFX1030_SUPPORT"}) + feature_names = {flag[2:].split("=")[0] for flag in feature_flags} + + def effective_features(flags, compiler): + result = subprocess.run([compiler, *flags, "-dM", "-E", "-x", "c++", "-"], + input="", capture_output=True, text=True) + assert result.returncode == 0, result.stderr + macros = {} + for line in result.stdout.splitlines(): + _, name, *value = line.split(maxsplit=2) + if name in feature_names: + macros[name] = value[0] if value else "" + return macros for op in OPS: command = next(c["command"] for c in commands if c["file"].endswith(f"grouped_gemm_{op}_ctypes_lib.cpp")) base = arch.split(":")[0] @@ -188,5 +212,8 @@ def test_cmake_normalizes_explicit_and_inferred_arches(tmp_path, explicit, arch) expected = set(arch_feature_defines(base)) if base == "gfx1250": expected.add("-DUSE_NEW_UNIFIED_FRAMEWORK=0") - actual = set(shlex.split(command)) & feature_flags - assert actual == expected, f"{op} {arch}: CMake/JIT feature mismatch: {actual ^ expected}" + args = shlex.split(command) + flags = [arg for arg in args[1:] if arg.startswith(("-D", "-U"))] + actual = effective_features(flags, args[0]) + expected = effective_features(sorted(expected), args[0]) + assert actual == expected, f"{op} {arch}: CMake/JIT feature mismatch: {actual} != {expected}" From 110c3eeb2b31434a8ae2db132a8ebb2f6f7531d8 Mon Sep 17 00:00:00 2001 From: Muhammed Emin Ozturk <3836908+ozturkosu@users.noreply.github.com> Date: Fri, 18 Sep 2026 03:58:06 -0400 Subject: [PATCH 24/24] fix(ck): require target for custom grouped quant tiles --- .../dispatcher/codegen/codegen_common.py | 10 ++++- ...nified_grouped_gemm_rowcolquant_codegen.py | 17 ++++++-- ...nified_grouped_gemm_tensorquant_codegen.py | 17 ++++++-- .../tests/test_quant_review_regressions.py | 40 +++++++++++++++++++ 4 files changed, 76 insertions(+), 8 deletions(-) diff --git a/projects/composablekernel/dispatcher/codegen/codegen_common.py b/projects/composablekernel/dispatcher/codegen/codegen_common.py index 0ae10e1785b5..c70dde62a156 100644 --- a/projects/composablekernel/dispatcher/codegen/codegen_common.py +++ b/projects/composablekernel/dispatcher/codegen/codegen_common.py @@ -1045,13 +1045,14 @@ def rowcol_tensor_quant_default_tile(gfx_arch: str = "") -> dict: ROWCOL_TENSOR_QUANT_SUPPORTED_ARCHES = ("gfx942", "gfx950", "gfx1250") -def validate_rowcol_tensor_quant_gfx_arch(gfx_arch: str) -> str: +def validate_rowcol_tensor_quant_gfx_arch(gfx_arch: str, *, require_explicit: bool = False) -> str: """Normalize and check a caller-supplied gfx target; return the bare target. Raises ``ValueError`` for anything outside ``ROWCOL_TENSOR_QUANT_SUPPORTED_ARCHES``. Empty is allowed and means "not specified", which selects the gfx9 MFMA tile -- the behaviour every invocation - without the flag had before the flag existed. + without the flag had before the flag existed. Custom tile configurations must + set ``require_explicit=True`` so their target check cannot be bypassed. This exists because ``--gfx-arch`` on the two codegen scripts is the one place a typo is completely silent. Everywhere else a bad target eventually reaches @@ -1061,6 +1062,11 @@ def validate_rowcol_tensor_quant_gfx_arch(gfx_arch: str) -> str: this whole branch exists to close, arriving through the front door. """ if not gfx_arch: + if require_explicit: + raise ValueError( + "Custom tile_configs require an explicit --gfx-arch (gfx_arch in Python) " + "so the generated header can reject a mismatched build target." + ) return "" base = normalize_gfx_arch(gfx_arch) if base not in ROWCOL_TENSOR_QUANT_SUPPORTED_ARCHES: diff --git a/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_rowcolquant_codegen.py b/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_rowcolquant_codegen.py index 7350b22f231f..a9fe03b2ddaa 100644 --- a/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_rowcolquant_codegen.py +++ b/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_rowcolquant_codegen.py @@ -550,9 +550,12 @@ def generate_kernels( """Generate all RowColQuant kernel headers into output_dir. Returns list of generated .hpp paths. `gfx_arch` selects the default tile and is embedded in every header so the - bridge can reject a mismatched build target. Explicit tiles remain unchanged. + bridge can reject a mismatched build target. Custom tile_configs require an + explicit target; their tile values remain unchanged. """ - gfx_arch = validate_rowcol_tensor_quant_gfx_arch(gfx_arch) + gfx_arch = validate_rowcol_tensor_quant_gfx_arch( + gfx_arch, require_explicit=config is not None and "tile_configs" in config + ) output_dir.mkdir(parents=True, exist_ok=True) cfg = config or _default_config(gfx_arch) specs = _build_specs(cfg) @@ -618,7 +621,7 @@ def main() -> int: help="GPU target the generated kernels will be compiled for, " "e.g. gfx1250. Embedded in generated headers; selects the default tile " "unless --config/--config-json supplies tile_configs. " - "Defaults to the gfx9 MFMA tile.") + "Required with custom tile_configs; otherwise defaults to the gfx9 MFMA tile.") args = parser.parse_args() # Validate before anything uses it. This value never reaches a compiler -- it @@ -642,6 +645,14 @@ def main() -> int: with open(args.config) as f: cfg = json.load(f) + try: + gfx_arch = validate_rowcol_tensor_quant_gfx_arch( + gfx_arch, require_explicit=cfg is not None and "tile_configs" in cfg + ) + except ValueError as e: + log.error("%s", e) + return 1 + if args.list_names: specs = _build_specs(cfg or _default_config(gfx_arch)) for s in specs: diff --git a/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_tensorquant_codegen.py b/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_tensorquant_codegen.py index 4031559c50af..bd63e47b79fb 100644 --- a/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_tensorquant_codegen.py +++ b/projects/composablekernel/dispatcher/codegen/unified_grouped_gemm_tensorquant_codegen.py @@ -550,9 +550,12 @@ def generate_kernels( """Generate all TensorQuant kernel headers into output_dir. Returns list of generated .hpp paths. `gfx_arch` selects the default tile and is embedded in every header so the - bridge can reject a mismatched build target. Explicit tiles remain unchanged. + bridge can reject a mismatched build target. Custom tile_configs require an + explicit target; their tile values remain unchanged. """ - gfx_arch = validate_rowcol_tensor_quant_gfx_arch(gfx_arch) + gfx_arch = validate_rowcol_tensor_quant_gfx_arch( + gfx_arch, require_explicit=config is not None and "tile_configs" in config + ) output_dir.mkdir(parents=True, exist_ok=True) cfg = config or _default_config(gfx_arch) specs = _build_specs(cfg) @@ -618,7 +621,7 @@ def main() -> int: help="GPU target the generated kernels will be compiled for, " "e.g. gfx1250. Embedded in generated headers; selects the default tile " "unless --config/--config-json supplies tile_configs. " - "Defaults to the gfx9 MFMA tile.") + "Required with custom tile_configs; otherwise defaults to the gfx9 MFMA tile.") args = parser.parse_args() # Validate before anything uses it. This value never reaches a compiler -- it @@ -642,6 +645,14 @@ def main() -> int: with open(args.config) as f: cfg = json.load(f) + try: + gfx_arch = validate_rowcol_tensor_quant_gfx_arch( + gfx_arch, require_explicit=cfg is not None and "tile_configs" in cfg + ) + except ValueError as e: + log.error("%s", e) + return 1 + if args.list_names: specs = _build_specs(cfg or _default_config(gfx_arch)) for s in specs: diff --git a/projects/composablekernel/dispatcher/tests/test_quant_review_regressions.py b/projects/composablekernel/dispatcher/tests/test_quant_review_regressions.py index b6f29fb33721..fda26cf79a56 100644 --- a/projects/composablekernel/dispatcher/tests/test_quant_review_regressions.py +++ b/projects/composablekernel/dispatcher/tests/test_quant_review_regressions.py @@ -21,6 +21,46 @@ CXX = shutil.which("c++") +@pytest.mark.parametrize("op", OPS) +def test_custom_quant_tiles_require_explicit_arch(op, tmp_path): + codegen = importlib.import_module(f"unified_grouped_gemm_{op}_codegen") + config = codegen._default_config("gfx1250") + output = tmp_path / "headers" + with pytest.raises(ValueError, match="explicit.*gfx.arch"): + codegen.generate_kernels(output, config=config, parallel=False) + assert not output.exists() + + # The same custom tile is valid when its target is explicitly recorded. + header = codegen.generate_kernels(output, config=config, gfx_arch="gfx1250:xnack-", + parallel=False)[0].read_text() + assert 'GfxArch = "gfx1250"' in header + # Preserve legacy no-config generation without a target. + legacy = codegen.generate_kernels(tmp_path / "legacy", parallel=False)[0].read_text() + assert 'GfxArch = ""' in legacy + assert "WarpTileM = 32" in legacy + + +@pytest.mark.parametrize("op", OPS) +@pytest.mark.parametrize("config_mode", ["--config", "--config-json"]) +@pytest.mark.parametrize("list_names", [False, True]) +def test_custom_quant_cli_requires_explicit_arch(op, config_mode, list_names, tmp_path): + codegen = importlib.import_module(f"unified_grouped_gemm_{op}_codegen") + config_json = json.dumps(codegen._default_config("gfx1250")) + config_file = tmp_path / "config.json" + config_file.write_text(config_json) + output = tmp_path / "headers" + args = [sys.executable, str(ROOT / f"codegen/unified_grouped_gemm_{op}_codegen.py"), + "--output-dir", str(output), config_mode, + str(config_file) if config_mode == "--config" else config_json] + if list_names: + args.append("--list-names") + result = subprocess.run(args, capture_output=True, text=True) + assert result.returncode != 0 + assert "explicit --gfx-arch" in result.stderr + assert "Traceback" not in result.stderr + assert not output.exists() + + @pytest.mark.parametrize("op", OPS) @pytest.mark.parametrize("config_arch,build_arch", [("gfx1250", "gfx942"), ("gfx942", "gfx1250")]) @pytest.mark.parametrize("explicit", [False, True])