From 5b8d025989635722534471dda1f3dc4665a715ab Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Mon, 17 Aug 2026 15:09:28 -0400 Subject: [PATCH 01/51] ci(ck-tile): add DISPATCHER CI stage for universal GEMM Adds a new 'Run DISPATCHER Tests' Jenkins stage parallel to the existing tile engine stages, targeting gfx942 with dispatcher_gemm_lib build and the tile_engine_dispatcher_bridge.py sweep runner. - Jenkinsfile: add RUN_DISPATCHER_TESTS bool param (default OFF) - Jenkinsfile: wire RUN_DISPATCHER_TESTS into the daily 22:00 cron - Jenkinsfile: add 'Run DISPATCHER Tests' stage on gfx942 - ck.groovy: add runDispatcherTests(arch, compiler) with DISPATCHER_RULE_SET=tests Co-Authored-By: Claude --- projects/composablekernel/Jenkinsfile | 34 ++++++++++++++++++- .../composablekernel/groovy/vars/ck.groovy | 18 ++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index 83992d20c8af..fe50811faaea 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -37,7 +37,7 @@ def loadCk() { //launch develop branch daily jobs CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;RUN_CK_TILE_FMHA_TESTS=true;RUN_PERFORMANCE_TESTS=true;FORCE_CI=true - 0 22 * * * % RUN_FULL_QA=true;DISABLE_DL_KERNELS=true;RUN_TILE_ENGINE_BASIC_TESTS=true;RUN_TILE_ENGINE_GEMM_TESTS=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true + 0 22 * * * % RUN_FULL_QA=true;DISABLE_DL_KERNELS=true;RUN_TILE_ENGINE_BASIC_TESTS=true;RUN_TILE_ENGINE_GEMM_TESTS=true;RUN_DISPATCHER_TESTS=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true 0 21 * * * % RUN_GROUPED_CONV_LARGE_CASES_TESTS=true;hipTensor_test=true;BUILD_GFX101=false;BUILD_GFX908=false;BUILD_GFX942=true;BUILD_GFX950=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true;BUILD_PACKAGES=true 0 19 * * * % BUILD_DOCKER=true;COMPILER_VERSION=develop;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true 0 17 * * * % BUILD_DOCKER=true;COMPILER_VERSION=therock;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true @@ -135,6 +135,10 @@ pipeline { name: "RUN_TILE_ENGINE_GEMM_TESTS", defaultValue: false, description: "Run the tile_engine_gemm tests (default: OFF)") + booleanParam( + name: "RUN_DISPATCHER_TESTS", + defaultValue: false, + description: "Run dispatcher universal GEMM tests (default: OFF)") booleanParam( name: "BUILD_INSTANCES_ONLY", defaultValue: false, @@ -672,6 +676,34 @@ pipeline { } } } + stage("Run DISPATCHER Tests") + { + when { + beforeAgent true + expression { env.SHOULD_RUN_CI.toBoolean() } + } + parallel + { + stage("Run DISPATCHER Tests on gfx942") + { + when { + beforeAgent true + expression { params.RUN_DISPATCHER_TESTS.toBoolean() } + } + agent none + steps{ + script { + loadCk() + ck.runOnHealthyNode(rocmnode("gfx942")) { + deleteDir() + ck.runDispatcherTests("gfx942", params.BUILD_COMPILER) + cleanWs() + } + } + } + } + } + } stage("Build CK and run Tests") { diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 554e3aec3d0d..65c1df8380b3 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1501,6 +1501,24 @@ def runTileEngineGemmTests(String arch, String compiler) { buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) } +def runDispatcherTests(String arch, String compiler) { + def execute_cmd = """ + cd build && \ + cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ + -D CMAKE_CXX_COMPILER="${compiler}" \ + -D CMAKE_BUILD_TYPE=Release \ + -D GPU_TARGETS="${arch}" \ + -D CK_TILE_DISPATCHER=ON \ + -D BUILD_DISPATCHER_BINDINGS=ON \ + -D DISPATCHER_RULE_SET=tests .. && \ + ninja -j${nthreads()} dispatcher_gemm_lib && \ + python3 ../dispatcher/examples/gemm/python/tile_engine_dispatcher_bridge.py \ + --demo sweep \ + --size 1024 \ + --arch ${arch}""" + buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) +} + def runBuildCKAndTests(String arch) { def gpuTarget def extraSetupArgs = "" From de5766ce3a45e4bd063e51f677e050cb598794f1 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Mon, 17 Aug 2026 15:30:47 -0400 Subject: [PATCH 02/51] ci(ck-tile): add dispatcher universal GEMM search space runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the 4-algo hardcoded sweep with a budget-limited stratified search over the full (dtype × layout × tile × trait) space, matching tile_engine's daily-tier coverage for universal GEMM. New files: - dispatcher/codegen/gemm_search_space.py: enumerate_configs wraps expand_sweep across all (dtype, layout) strata; sample_configs does stratified budget-limited sampling with daily rotating seed. - dispatcher/tests/run_gemm_search_space.py: CLI runner -- enumerate, sample, parallel JIT build, GPU run, correctness check, JSON results. Updated: - dispatcher/CMakeLists.txt: add DISPATCHER_GEMM_BUDGET cache var (500). - groovy/vars/ck.groovy: runDispatcherTests now calls run_gemm_search_space.py with --budget 500, all dtypes and layouts, writing dispatcher_gemm_results.json as an artifact. Coverage after this change: ~500 stratified configs across fp16/bf16/fp8/bf8 × rcr/rrr/crr/ccr (vs 4 hardcoded before). Co-Authored-By: Claude --- .../dispatcher/CMakeLists.txt | 5 + .../dispatcher/codegen/gemm_search_space.py | 131 +++++++++++ .../dispatcher/tests/run_gemm_search_space.py | 208 ++++++++++++++++++ .../composablekernel/groovy/vars/ck.groovy | 12 +- 4 files changed, 352 insertions(+), 4 deletions(-) create mode 100644 projects/composablekernel/dispatcher/codegen/gemm_search_space.py create mode 100644 projects/composablekernel/dispatcher/tests/run_gemm_search_space.py diff --git a/projects/composablekernel/dispatcher/CMakeLists.txt b/projects/composablekernel/dispatcher/CMakeLists.txt index 79bdde45e87e..04b324f22171 100644 --- a/projects/composablekernel/dispatcher/CMakeLists.txt +++ b/projects/composablekernel/dispatcher/CMakeLists.txt @@ -74,6 +74,11 @@ if(BUILD_DISPATCHER_TESTS) add_subdirectory(tests) endif() +# CI search-space budget for run_gemm_search_space.py. +# 0 = no cap (full space); matches tile_engine's TILE_ENGINE_SAMPLING_TIER=daily (~500/op). +set(DISPATCHER_GEMM_BUDGET "500" CACHE STRING + "Max GemmKernelConfig instances for CI search-space runner (0 = no cap)") + # Optional: Build Python bindings option(BUILD_DISPATCHER_PYTHON "Build Python bindings for dispatcher" OFF) if(BUILD_DISPATCHER_PYTHON) diff --git a/projects/composablekernel/dispatcher/codegen/gemm_search_space.py b/projects/composablekernel/dispatcher/codegen/gemm_search_space.py new file mode 100644 index 000000000000..b5eebbfd2a20 --- /dev/null +++ b/projects/composablekernel/dispatcher/codegen/gemm_search_space.py @@ -0,0 +1,131 @@ +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +""" +Dispatcher universal GEMM search space: enumerate, sample, and export configs. + +Wraps gemm_utils.expand_sweep to produce a stratified, budget-limited set of +GemmKernelConfig objects across all supported (dtype, layout) combinations, +mirroring tile_engine's daily-tier sampling for CI coverage. + +Public surface: + DEFAULT_CI_CONFIG -- path to tile_engine's default_ci_config.json + DTYPES -- supported dtype list (matches tile engine gfx942 set) + LAYOUTS -- supported layout list + enumerate_configs -- full search space across all dtype/layout combos + sample_configs -- budget-limited stratified subset with seed support + daily_seed -- date-based rotating seed (mirrors tile engine convention) +""" + +from __future__ import annotations + +import hashlib +import random +from datetime import date +from pathlib import Path +from typing import Dict, List, Optional + +# Path to tile engine's CI config -- reused directly so dispatcher and tile +# engine sweep the same tile/trait dimensions without duplication. +DEFAULT_CI_CONFIG = ( + Path(__file__).parent.parent.parent + / "tile_engine/ops/gemm/configs/default_ci_config.json" +) + +# Supported dtypes on gfx942 for universal GEMM (mirrors tile engine CI flags). +DTYPES = ["fp16", "bf16", "fp8", "bf8"] + +# Supported layout combos (A-major, B-major, C-major). +# Column-major C is rejected by the universal GEMM codegen, so all four +# A/B combinations keep row-major C -- matching tile engine's layout flags. +LAYOUTS = ["rcr", "rrr", "crr", "ccr"] + + +def enumerate_configs( + arch: str, + dtypes: Optional[List[str]] = None, + layouts: Optional[List[str]] = None, + config_path: Optional[str] = None, +) -> Dict[str, list]: + """Enumerate the full search space, grouped by (dtype, layout) stratum. + + Args: + arch: GPU architecture string (e.g. 'gfx942'). + dtypes: Dtype list to sweep; defaults to DTYPES. + layouts: Layout list to sweep; defaults to LAYOUTS. + config_path: Path to a TE-format config JSON; defaults to DEFAULT_CI_CONFIG. + + Returns: + Dict mapping (dtype, layout) tuple-keys to lists of GemmKernelConfig. + Key format: "/" (e.g. "fp16/rcr"). + """ + import sys + + sys.path.insert(0, str(Path(__file__).parent.parent / "python")) + from gemm_utils import expand_sweep + + cfg = str(config_path or DEFAULT_CI_CONFIG) + dtypes = dtypes or DTYPES + layouts = layouts or LAYOUTS + + strata: Dict[str, list] = {} + for dtype in dtypes: + for layout in layouts: + key = f"{dtype}/{layout}" + configs = expand_sweep(cfg, arch=arch, dtype=dtype, layout=layout) + strata[key] = configs + + return strata + + +def daily_seed() -> int: + """Return a date-based integer seed that rotates daily. + + Mirrors tile_engine's daily seed rotation so the sampled subset changes + each day, giving broad coverage over time without a fixed bias. + """ + today = date.today().isoformat() + return int(hashlib.md5(today.encode()).hexdigest(), 16) % (2**31) + + +def sample_configs( + strata: Dict[str, list], + budget: int, + seed: Optional[int] = None, +) -> list: + """Stratified budget-limited sample from the search space. + + Distributes the budget equally across all non-empty (dtype, layout) strata, + then samples randomly within each stratum. If a stratum has fewer configs + than its share, all are included and the remainder is redistributed. + + Args: + strata: Output of enumerate_configs -- dict of stratum -> config list. + budget: Total number of configs to return (hard cap). + seed: RNG seed. Pass None to use daily_seed() for daily rotation. + + Returns: + Flat list of GemmKernelConfig, length <= budget. + """ + if seed is None: + seed = daily_seed() + + rng = random.Random(seed) + + nonempty = {k: v for k, v in strata.items() if v} + if not nonempty: + return [] + + n_strata = len(nonempty) + per_stratum = max(1, budget // n_strata) + remainder = budget - per_stratum * n_strata + + selected = [] + for i, (key, configs) in enumerate(nonempty.items()): + # Last stratum absorbs the rounding remainder. + alloc = per_stratum + (remainder if i == n_strata - 1 else 0) + alloc = min(alloc, len(configs)) + chosen = rng.sample(configs, alloc) + selected.extend(chosen) + + rng.shuffle(selected) + return selected[:budget] diff --git a/projects/composablekernel/dispatcher/tests/run_gemm_search_space.py b/projects/composablekernel/dispatcher/tests/run_gemm_search_space.py new file mode 100644 index 000000000000..d0cbc97db450 --- /dev/null +++ b/projects/composablekernel/dispatcher/tests/run_gemm_search_space.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 + +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +""" +Dispatcher universal GEMM search space runner. + +Enumerates the full GemmKernelConfig search space (mirroring tile engine's +daily-tier coverage), samples a budget-limited subset, compiles each config +via the bridge, runs on GPU, and reports correctness + TFLOPS. + +Usage: + python3 run_gemm_search_space.py --arch gfx942 --budget 500 + python3 run_gemm_search_space.py --dtypes fp16,bf16 --layouts rcr,rrr + python3 run_gemm_search_space.py --budget 500 --seed 42 --json results.json + python3 run_gemm_search_space.py --budget 0 # run full search space (no cap) +""" + +import argparse +import json +import sys +import time +from pathlib import Path + +# Ensure dispatcher python and codegen are on path. +_DISPATCHER = Path(__file__).parent.parent +sys.path.insert(0, str(_DISPATCHER / "python")) +sys.path.insert(0, str(_DISPATCHER / "codegen")) + +import numpy as np + +from ctypes_utils import detect_gpu_arch +from gemm_utils import GemmProblem, GpuGemmRunner, setup_multiple_gemm_dispatchers +from gemm_search_space import ( + DTYPES, + LAYOUTS, + daily_seed, + enumerate_configs, + sample_configs, +) + +_RTOL = 2e-2 + + +def _emulate(x: np.ndarray, dtype: str) -> np.ndarray: + if dtype == "bf16": + u32 = np.ascontiguousarray(x, dtype=np.float32).view(np.uint32) + rounded = (u32 + ((u32 >> 16) & 1) + np.uint32(0x7FFF)) >> 16 + return (rounded.astype(np.uint32) << 16).view(np.float32) + if dtype in ("fp8", "bf8"): + # Approximate: cast to fp16 precision for reference. + return x.astype(np.float16).astype(np.float32) + return x.astype(np.float16).astype(np.float32) + + +def _reference(A, B, dtype): + return _emulate(_emulate(A, dtype) @ _emulate(B, dtype), dtype) + + +def _max_rel(out: np.ndarray, ref: np.ndarray) -> float: + denom = float(np.max(np.abs(ref))) + 1e-12 + return float(np.max(np.abs(out - ref))) / denom + + +def _np_dtype(dtype: str) -> np.dtype: + if dtype in ("fp16", "bf16"): + return np.float16 + if dtype in ("fp8", "bf8"): + return np.uint8 + return np.float32 + + +def run(args) -> int: + arch = args.arch or detect_gpu_arch() + dtypes = [d.strip() for d in args.dtypes.split(",")] + layouts = [l.strip() for l in args.layouts.split(",")] + budget = args.budget + seed = args.seed if args.seed is not None else daily_seed() + size = args.size + + print(f"arch={arch} dtypes={dtypes} layouts={layouts}") + print(f"budget={budget if budget > 0 else 'unlimited'} seed={seed} size={size}") + + # --- Enumerate --- + print("\nEnumerating search space...") + t0 = time.time() + strata = enumerate_configs(arch=arch, dtypes=dtypes, layouts=layouts) + total_configs = sum(len(v) for v in strata.values()) + print(f" {total_configs} valid configs across {len(strata)} strata " + f"({time.time()-t0:.1f}s)") + for key, cfgs in sorted(strata.items()): + print(f" {key}: {len(cfgs)}") + + # --- Sample --- + if budget > 0 and budget < total_configs: + configs = sample_configs(strata, budget=budget, seed=seed) + print(f"\nSampled {len(configs)}/{total_configs} configs (budget={budget})") + else: + configs = [c for v in strata.values() for c in v] + print(f"\nRunning all {len(configs)} configs (no budget cap)") + + if not configs: + print("ERROR: no configs to run") + return 1 + + # --- Build --- + print(f"\nBuilding {len(configs)} .so files (parallel JIT)...") + t0 = time.time() + so_paths = setup_multiple_gemm_dispatchers(configs, verbose=False) + build_ok = sum(1 for p in so_paths if p is not None) + build_fail = len(so_paths) - build_ok + print(f" Built {build_ok}/{len(configs)} in {time.time()-t0:.1f}s " + f"({build_fail} failed)") + + # --- Run --- + print(f"\nRunning {build_ok} kernels on GPU...") + problem = GemmProblem(M=size, N=size, K=size) + rng = np.random.default_rng(seed) + A = (rng.standard_normal((size, size)) * 0.1).astype(np.float32) + B = (rng.standard_normal((size, size)) * 0.1).astype(np.float32) + + results = [] + n_pass = n_fail = n_build_fail = 0 + + for cfg, so in zip(configs, so_paths): + dtype = cfg.dtype_a + label = cfg.name + + if so is None: + n_build_fail += 1 + results.append({"name": label, "status": "build_fail"}) + continue + + try: + runner = GpuGemmRunner(lib_path=so) + result = runner.run(A, B, problem) + except Exception as exc: + n_fail += 1 + results.append({"name": label, "status": "run_error", "error": str(exc)}) + continue + + if not result.success: + n_fail += 1 + results.append({"name": label, "status": "run_fail", + "error": f"status={result.status}"}) + continue + + ref = _reference(A, B, dtype) + mr = _max_rel(result.output, ref) + ok = mr <= _RTOL + n_pass += ok + n_fail += not ok + results.append({ + "name": label, + "status": "pass" if ok else "fail", + "tflops": round(result.tflops, 3), + "max_rel": round(mr, 6), + }) + + # --- Report --- + print(f"\n{'='*60}") + print(f"Results: {n_pass} pass / {n_fail} fail / {n_build_fail} build-fail " + f"/ {len(configs)} total") + + if args.json: + out = { + "arch": arch, "size": size, "seed": seed, + "budget": budget, "total_configs": total_configs, + "n_pass": n_pass, "n_fail": n_fail, "n_build_fail": n_build_fail, + "kernels": results, + } + Path(args.json).write_text(json.dumps(out, indent=2)) + print(f"Results written to {args.json}") + + if n_fail > 0: + print("\nFailed kernels:") + for r in results: + if r["status"] not in ("pass", "build_fail"): + print(f" {r['name']}: {r['status']} " + f"{r.get('error', r.get('max_rel', ''))}") + + return 0 if n_fail == 0 else 1 + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Dispatcher universal GEMM search space runner", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--arch", default=None, + help="GPU architecture (default: auto-detect via rocminfo)") + parser.add_argument("--dtypes", default=",".join(DTYPES), + help=f"Comma-separated dtypes (default: {','.join(DTYPES)})") + parser.add_argument("--layouts", default=",".join(LAYOUTS), + help=f"Comma-separated layouts (default: {','.join(LAYOUTS)})") + parser.add_argument("--budget", type=int, default=500, + help="Max configs to run; 0 = no cap (default: 500)") + parser.add_argument("--seed", type=int, default=None, + help="RNG seed; default = daily rotating seed") + parser.add_argument("--size", type=int, default=1024, + help="M=N=K problem size (default: 1024)") + parser.add_argument("--json", default=None, + help="Write results JSON to this path") + return run(parser.parse_args()) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 65c1df8380b3..ee3acbf06d2e 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1510,12 +1510,16 @@ def runDispatcherTests(String arch, String compiler) { -D GPU_TARGETS="${arch}" \ -D CK_TILE_DISPATCHER=ON \ -D BUILD_DISPATCHER_BINDINGS=ON \ - -D DISPATCHER_RULE_SET=tests .. && \ + -D DISPATCHER_RULE_SET=tests \ + -D DISPATCHER_GEMM_BUDGET=500 .. && \ ninja -j${nthreads()} dispatcher_gemm_lib && \ - python3 ../dispatcher/examples/gemm/python/tile_engine_dispatcher_bridge.py \ - --demo sweep \ + python3 ../dispatcher/tests/run_gemm_search_space.py \ + --arch ${arch} \ + --dtypes fp16,bf16,fp8,bf8 \ + --layouts rcr,rrr,crr,ccr \ + --budget 500 \ --size 1024 \ - --arch ${arch}""" + --json dispatcher_gemm_results.json""" buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) } From 538f4ebdc4f3ee934d3828c231460c805cc38c01 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Mon, 17 Aug 2026 15:47:30 -0400 Subject: [PATCH 03/51] refactor(ck-tile): inline search space logic into runner, remove extra file gemm_search_space.py was an unnecessary indirection -- tile engine has no equivalent; sampling lives inside the instance builder itself. Move the _daily_seed and _sample helpers directly into run_gemm_search_space.py and call expand_sweep from gemm_utils directly, matching the tile engine pattern more closely. Co-Authored-By: Claude --- .../dispatcher/codegen/gemm_search_space.py | 131 -------------- .../dispatcher/tests/run_gemm_search_space.py | 165 +++++++++--------- 2 files changed, 81 insertions(+), 215 deletions(-) delete mode 100644 projects/composablekernel/dispatcher/codegen/gemm_search_space.py diff --git a/projects/composablekernel/dispatcher/codegen/gemm_search_space.py b/projects/composablekernel/dispatcher/codegen/gemm_search_space.py deleted file mode 100644 index b5eebbfd2a20..000000000000 --- a/projects/composablekernel/dispatcher/codegen/gemm_search_space.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. -# SPDX-License-Identifier: MIT -""" -Dispatcher universal GEMM search space: enumerate, sample, and export configs. - -Wraps gemm_utils.expand_sweep to produce a stratified, budget-limited set of -GemmKernelConfig objects across all supported (dtype, layout) combinations, -mirroring tile_engine's daily-tier sampling for CI coverage. - -Public surface: - DEFAULT_CI_CONFIG -- path to tile_engine's default_ci_config.json - DTYPES -- supported dtype list (matches tile engine gfx942 set) - LAYOUTS -- supported layout list - enumerate_configs -- full search space across all dtype/layout combos - sample_configs -- budget-limited stratified subset with seed support - daily_seed -- date-based rotating seed (mirrors tile engine convention) -""" - -from __future__ import annotations - -import hashlib -import random -from datetime import date -from pathlib import Path -from typing import Dict, List, Optional - -# Path to tile engine's CI config -- reused directly so dispatcher and tile -# engine sweep the same tile/trait dimensions without duplication. -DEFAULT_CI_CONFIG = ( - Path(__file__).parent.parent.parent - / "tile_engine/ops/gemm/configs/default_ci_config.json" -) - -# Supported dtypes on gfx942 for universal GEMM (mirrors tile engine CI flags). -DTYPES = ["fp16", "bf16", "fp8", "bf8"] - -# Supported layout combos (A-major, B-major, C-major). -# Column-major C is rejected by the universal GEMM codegen, so all four -# A/B combinations keep row-major C -- matching tile engine's layout flags. -LAYOUTS = ["rcr", "rrr", "crr", "ccr"] - - -def enumerate_configs( - arch: str, - dtypes: Optional[List[str]] = None, - layouts: Optional[List[str]] = None, - config_path: Optional[str] = None, -) -> Dict[str, list]: - """Enumerate the full search space, grouped by (dtype, layout) stratum. - - Args: - arch: GPU architecture string (e.g. 'gfx942'). - dtypes: Dtype list to sweep; defaults to DTYPES. - layouts: Layout list to sweep; defaults to LAYOUTS. - config_path: Path to a TE-format config JSON; defaults to DEFAULT_CI_CONFIG. - - Returns: - Dict mapping (dtype, layout) tuple-keys to lists of GemmKernelConfig. - Key format: "/" (e.g. "fp16/rcr"). - """ - import sys - - sys.path.insert(0, str(Path(__file__).parent.parent / "python")) - from gemm_utils import expand_sweep - - cfg = str(config_path or DEFAULT_CI_CONFIG) - dtypes = dtypes or DTYPES - layouts = layouts or LAYOUTS - - strata: Dict[str, list] = {} - for dtype in dtypes: - for layout in layouts: - key = f"{dtype}/{layout}" - configs = expand_sweep(cfg, arch=arch, dtype=dtype, layout=layout) - strata[key] = configs - - return strata - - -def daily_seed() -> int: - """Return a date-based integer seed that rotates daily. - - Mirrors tile_engine's daily seed rotation so the sampled subset changes - each day, giving broad coverage over time without a fixed bias. - """ - today = date.today().isoformat() - return int(hashlib.md5(today.encode()).hexdigest(), 16) % (2**31) - - -def sample_configs( - strata: Dict[str, list], - budget: int, - seed: Optional[int] = None, -) -> list: - """Stratified budget-limited sample from the search space. - - Distributes the budget equally across all non-empty (dtype, layout) strata, - then samples randomly within each stratum. If a stratum has fewer configs - than its share, all are included and the remainder is redistributed. - - Args: - strata: Output of enumerate_configs -- dict of stratum -> config list. - budget: Total number of configs to return (hard cap). - seed: RNG seed. Pass None to use daily_seed() for daily rotation. - - Returns: - Flat list of GemmKernelConfig, length <= budget. - """ - if seed is None: - seed = daily_seed() - - rng = random.Random(seed) - - nonempty = {k: v for k, v in strata.items() if v} - if not nonempty: - return [] - - n_strata = len(nonempty) - per_stratum = max(1, budget // n_strata) - remainder = budget - per_stratum * n_strata - - selected = [] - for i, (key, configs) in enumerate(nonempty.items()): - # Last stratum absorbs the rounding remainder. - alloc = per_stratum + (remainder if i == n_strata - 1 else 0) - alloc = min(alloc, len(configs)) - chosen = rng.sample(configs, alloc) - selected.extend(chosen) - - rng.shuffle(selected) - return selected[:budget] diff --git a/projects/composablekernel/dispatcher/tests/run_gemm_search_space.py b/projects/composablekernel/dispatcher/tests/run_gemm_search_space.py index d0cbc97db450..197b16f3c886 100644 --- a/projects/composablekernel/dispatcher/tests/run_gemm_search_space.py +++ b/projects/composablekernel/dispatcher/tests/run_gemm_search_space.py @@ -5,69 +5,77 @@ """ Dispatcher universal GEMM search space runner. -Enumerates the full GemmKernelConfig search space (mirroring tile engine's -daily-tier coverage), samples a budget-limited subset, compiles each config -via the bridge, runs on GPU, and reports correctness + TFLOPS. +Enumerates the full GemmKernelConfig search space by calling expand_sweep +with the tile_engine default_ci_config.json across all (dtype, layout) strata, +samples a budget-limited subset with a daily rotating seed, compiles each +config via the bridge, runs on GPU, and reports correctness + TFLOPS. + +This is the dispatcher equivalent of tile_engine's gemm_universal_benchmark.py +driven with TILE_ENGINE_SAMPLING_TIER=daily. Usage: python3 run_gemm_search_space.py --arch gfx942 --budget 500 python3 run_gemm_search_space.py --dtypes fp16,bf16 --layouts rcr,rrr + python3 run_gemm_search_space.py --budget 0 # full space, no cap python3 run_gemm_search_space.py --budget 500 --seed 42 --json results.json - python3 run_gemm_search_space.py --budget 0 # run full search space (no cap) """ import argparse +import hashlib import json +import random import sys import time +from datetime import date from pathlib import Path -# Ensure dispatcher python and codegen are on path. _DISPATCHER = Path(__file__).parent.parent sys.path.insert(0, str(_DISPATCHER / "python")) -sys.path.insert(0, str(_DISPATCHER / "codegen")) import numpy as np from ctypes_utils import detect_gpu_arch -from gemm_utils import GemmProblem, GpuGemmRunner, setup_multiple_gemm_dispatchers -from gemm_search_space import ( - DTYPES, - LAYOUTS, - daily_seed, - enumerate_configs, - sample_configs, -) +from gemm_utils import GemmProblem, GpuGemmRunner, expand_sweep, setup_multiple_gemm_dispatchers +_CI_CONFIG = ( + _DISPATCHER.parent / "tile_engine/ops/gemm/configs/default_ci_config.json" +) +_DTYPES = ["fp16", "bf16", "fp8", "bf8"] +_LAYOUTS = ["rcr", "rrr", "crr", "ccr"] _RTOL = 2e-2 +def _daily_seed() -> int: + return int(hashlib.md5(date.today().isoformat().encode()).hexdigest(), 16) % (2**31) + + +def _sample(strata: dict, budget: int, seed: int) -> list: + """Stratified budget-limited sample: equal share per non-empty stratum.""" + nonempty = {k: v for k, v in strata.items() if v} + if not nonempty: + return [] + rng = random.Random(seed) + n = len(nonempty) + per = max(1, budget // n) + remainder = budget - per * n + selected = [] + for i, (_, configs) in enumerate(nonempty.items()): + alloc = min(per + (remainder if i == n - 1 else 0), len(configs)) + selected.extend(rng.sample(configs, alloc)) + rng.shuffle(selected) + return selected[:budget] + + def _emulate(x: np.ndarray, dtype: str) -> np.ndarray: if dtype == "bf16": u32 = np.ascontiguousarray(x, dtype=np.float32).view(np.uint32) rounded = (u32 + ((u32 >> 16) & 1) + np.uint32(0x7FFF)) >> 16 return (rounded.astype(np.uint32) << 16).view(np.float32) - if dtype in ("fp8", "bf8"): - # Approximate: cast to fp16 precision for reference. - return x.astype(np.float16).astype(np.float32) return x.astype(np.float16).astype(np.float32) -def _reference(A, B, dtype): - return _emulate(_emulate(A, dtype) @ _emulate(B, dtype), dtype) - - def _max_rel(out: np.ndarray, ref: np.ndarray) -> float: - denom = float(np.max(np.abs(ref))) + 1e-12 - return float(np.max(np.abs(out - ref))) / denom - - -def _np_dtype(dtype: str) -> np.dtype: - if dtype in ("fp16", "bf16"): - return np.float16 - if dtype in ("fp8", "bf8"): - return np.uint8 - return np.float32 + return float(np.max(np.abs(out - ref))) / (float(np.max(np.abs(ref))) + 1e-12) def run(args) -> int: @@ -75,29 +83,33 @@ def run(args) -> int: dtypes = [d.strip() for d in args.dtypes.split(",")] layouts = [l.strip() for l in args.layouts.split(",")] budget = args.budget - seed = args.seed if args.seed is not None else daily_seed() + seed = args.seed if args.seed is not None else _daily_seed() size = args.size print(f"arch={arch} dtypes={dtypes} layouts={layouts}") print(f"budget={budget if budget > 0 else 'unlimited'} seed={seed} size={size}") - # --- Enumerate --- + # --- Enumerate: same JSON config as tile_engine, across all strata --- print("\nEnumerating search space...") t0 = time.time() - strata = enumerate_configs(arch=arch, dtypes=dtypes, layouts=layouts) - total_configs = sum(len(v) for v in strata.values()) - print(f" {total_configs} valid configs across {len(strata)} strata " - f"({time.time()-t0:.1f}s)") + strata = {} + for dtype in dtypes: + for layout in layouts: + configs = expand_sweep(str(_CI_CONFIG), arch=arch, + dtype=dtype, layout=layout) + strata[f"{dtype}/{layout}"] = configs + total = sum(len(v) for v in strata.values()) + print(f" {total} valid configs across {len(strata)} strata ({time.time()-t0:.1f}s)") for key, cfgs in sorted(strata.items()): print(f" {key}: {len(cfgs)}") # --- Sample --- - if budget > 0 and budget < total_configs: - configs = sample_configs(strata, budget=budget, seed=seed) - print(f"\nSampled {len(configs)}/{total_configs} configs (budget={budget})") + if budget > 0 and budget < total: + configs = _sample(strata, budget=budget, seed=seed) + print(f"\nSampled {len(configs)}/{total} configs (budget={budget})") else: configs = [c for v in strata.values() for c in v] - print(f"\nRunning all {len(configs)} configs (no budget cap)") + print(f"\nRunning all {len(configs)} configs (no cap)") if not configs: print("ERROR: no configs to run") @@ -108,12 +120,11 @@ def run(args) -> int: t0 = time.time() so_paths = setup_multiple_gemm_dispatchers(configs, verbose=False) build_ok = sum(1 for p in so_paths if p is not None) - build_fail = len(so_paths) - build_ok print(f" Built {build_ok}/{len(configs)} in {time.time()-t0:.1f}s " - f"({build_fail} failed)") + f"({len(configs)-build_ok} failed)") # --- Run --- - print(f"\nRunning {build_ok} kernels on GPU...") + print(f"\nRunning {build_ok} kernels on GPU (M=N=K={size})...") problem = GemmProblem(M=size, N=size, K=size) rng = np.random.default_rng(seed) A = (rng.standard_normal((size, size)) * 0.1).astype(np.float32) @@ -123,39 +134,28 @@ def run(args) -> int: n_pass = n_fail = n_build_fail = 0 for cfg, so in zip(configs, so_paths): - dtype = cfg.dtype_a - label = cfg.name - if so is None: n_build_fail += 1 - results.append({"name": label, "status": "build_fail"}) + results.append({"name": cfg.name, "status": "build_fail"}) continue - try: - runner = GpuGemmRunner(lib_path=so) - result = runner.run(A, B, problem) + result = GpuGemmRunner(lib_path=so).run(A, B, problem) except Exception as exc: n_fail += 1 - results.append({"name": label, "status": "run_error", "error": str(exc)}) + results.append({"name": cfg.name, "status": "run_error", "error": str(exc)}) continue - if not result.success: n_fail += 1 - results.append({"name": label, "status": "run_fail", + results.append({"name": cfg.name, "status": "run_fail", "error": f"status={result.status}"}) continue - - ref = _reference(A, B, dtype) + ref = _emulate(_emulate(A, cfg.dtype_a) @ _emulate(B, cfg.dtype_a), cfg.dtype_a) mr = _max_rel(result.output, ref) ok = mr <= _RTOL n_pass += ok n_fail += not ok - results.append({ - "name": label, - "status": "pass" if ok else "fail", - "tflops": round(result.tflops, 3), - "max_rel": round(mr, 6), - }) + results.append({"name": cfg.name, "status": "pass" if ok else "fail", + "tflops": round(result.tflops, 3), "max_rel": round(mr, 6)}) # --- Report --- print(f"\n{'='*60}") @@ -163,12 +163,9 @@ def run(args) -> int: f"/ {len(configs)} total") if args.json: - out = { - "arch": arch, "size": size, "seed": seed, - "budget": budget, "total_configs": total_configs, - "n_pass": n_pass, "n_fail": n_fail, "n_build_fail": n_build_fail, - "kernels": results, - } + out = {"arch": arch, "size": size, "seed": seed, "budget": budget, + "total_configs": total, "n_pass": n_pass, "n_fail": n_fail, + "n_build_fail": n_build_fail, "kernels": results} Path(args.json).write_text(json.dumps(out, indent=2)) print(f"Results written to {args.json}") @@ -183,25 +180,25 @@ def run(args) -> int: def main() -> int: - parser = argparse.ArgumentParser( + p = argparse.ArgumentParser( description="Dispatcher universal GEMM search space runner", formatter_class=argparse.RawDescriptionHelpFormatter, ) - parser.add_argument("--arch", default=None, - help="GPU architecture (default: auto-detect via rocminfo)") - parser.add_argument("--dtypes", default=",".join(DTYPES), - help=f"Comma-separated dtypes (default: {','.join(DTYPES)})") - parser.add_argument("--layouts", default=",".join(LAYOUTS), - help=f"Comma-separated layouts (default: {','.join(LAYOUTS)})") - parser.add_argument("--budget", type=int, default=500, - help="Max configs to run; 0 = no cap (default: 500)") - parser.add_argument("--seed", type=int, default=None, - help="RNG seed; default = daily rotating seed") - parser.add_argument("--size", type=int, default=1024, - help="M=N=K problem size (default: 1024)") - parser.add_argument("--json", default=None, - help="Write results JSON to this path") - return run(parser.parse_args()) + p.add_argument("--arch", default=None, + help="GPU arch (default: auto-detect via rocminfo)") + p.add_argument("--dtypes", default=",".join(_DTYPES), + help=f"Comma-separated dtypes (default: {','.join(_DTYPES)})") + p.add_argument("--layouts", default=",".join(_LAYOUTS), + help=f"Comma-separated layouts (default: {','.join(_LAYOUTS)})") + p.add_argument("--budget", type=int, default=500, + help="Max configs to run; 0 = no cap (default: 500)") + p.add_argument("--seed", type=int, default=None, + help="RNG seed; default = daily rotating seed") + p.add_argument("--size", type=int, default=1024, + help="M=N=K problem size (default: 1024)") + p.add_argument("--json", default=None, + help="Write results JSON to this path") + return run(p.parse_args()) if __name__ == "__main__": From f5288f4e0e549f1118932f84290a370e455dc02b Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Mon, 17 Aug 2026 15:49:09 -0400 Subject: [PATCH 04/51] fix(ck-tile): remove unused DISPATCHER_GEMM_BUDGET CMake var Budget is passed directly as --budget CLI arg to run_gemm_search_space.py from the groovy script; CMake is not in the loop for sampling, so the cache variable was dead code. Co-Authored-By: Claude --- projects/composablekernel/dispatcher/CMakeLists.txt | 5 ----- projects/composablekernel/groovy/vars/ck.groovy | 3 +-- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/projects/composablekernel/dispatcher/CMakeLists.txt b/projects/composablekernel/dispatcher/CMakeLists.txt index 04b324f22171..79bdde45e87e 100644 --- a/projects/composablekernel/dispatcher/CMakeLists.txt +++ b/projects/composablekernel/dispatcher/CMakeLists.txt @@ -74,11 +74,6 @@ if(BUILD_DISPATCHER_TESTS) add_subdirectory(tests) endif() -# CI search-space budget for run_gemm_search_space.py. -# 0 = no cap (full space); matches tile_engine's TILE_ENGINE_SAMPLING_TIER=daily (~500/op). -set(DISPATCHER_GEMM_BUDGET "500" CACHE STRING - "Max GemmKernelConfig instances for CI search-space runner (0 = no cap)") - # Optional: Build Python bindings option(BUILD_DISPATCHER_PYTHON "Build Python bindings for dispatcher" OFF) if(BUILD_DISPATCHER_PYTHON) diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index ee3acbf06d2e..01ebd04aa6da 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1510,8 +1510,7 @@ def runDispatcherTests(String arch, String compiler) { -D GPU_TARGETS="${arch}" \ -D CK_TILE_DISPATCHER=ON \ -D BUILD_DISPATCHER_BINDINGS=ON \ - -D DISPATCHER_RULE_SET=tests \ - -D DISPATCHER_GEMM_BUDGET=500 .. && \ + -D DISPATCHER_RULE_SET=tests .. && \ ninja -j${nthreads()} dispatcher_gemm_lib && \ python3 ../dispatcher/tests/run_gemm_search_space.py \ --arch ${arch} \ From e62b9ddb196fdf1b730c8379d388bb6e89f0a91b Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Mon, 17 Aug 2026 16:40:05 -0400 Subject: [PATCH 05/51] fix(ck-tile): fix dispatcher CI stage correctness issues Three fixes: 1. Remove spurious 'cd build &&' -- cmake_build already cds into build/ before running execute_cmd; the extra cd would fail (build/build/). 2. Add archiveArtifacts for dispatcher_gemm_results.json to match tile_engine's JSON artifact pattern. 3. Add --warmup/--repeat args to run_gemm_search_space.py (default 5/5) and wire them in from the groovy call, matching tile_engine convention. Co-Authored-By: Claude --- .../dispatcher/tests/run_gemm_search_space.py | 25 +++++++++++++++---- .../composablekernel/groovy/vars/ck.groovy | 4 ++- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/projects/composablekernel/dispatcher/tests/run_gemm_search_space.py b/projects/composablekernel/dispatcher/tests/run_gemm_search_space.py index 197b16f3c886..5c1ce8164a54 100644 --- a/projects/composablekernel/dispatcher/tests/run_gemm_search_space.py +++ b/projects/composablekernel/dispatcher/tests/run_gemm_search_space.py @@ -124,7 +124,8 @@ def run(args) -> int: f"({len(configs)-build_ok} failed)") # --- Run --- - print(f"\nRunning {build_ok} kernels on GPU (M=N=K={size})...") + print(f"\nRunning {build_ok} kernels on GPU (M=N=K={size}, " + f"warmup={args.warmup}, repeat={args.repeat})...") problem = GemmProblem(M=size, N=size, K=size) rng = np.random.default_rng(seed) A = (rng.standard_normal((size, size)) * 0.1).astype(np.float32) @@ -139,23 +140,33 @@ def run(args) -> int: results.append({"name": cfg.name, "status": "build_fail"}) continue try: - result = GpuGemmRunner(lib_path=so).run(A, B, problem) + runner = GpuGemmRunner(lib_path=so) + for _ in range(args.warmup): + runner.run(A, B, problem) + times = [] + result = None + for _ in range(max(1, args.repeat)): + result = runner.run(A, B, problem) + if result.success: + times.append(result.time_ms) except Exception as exc: n_fail += 1 results.append({"name": cfg.name, "status": "run_error", "error": str(exc)}) continue - if not result.success: + if result is None or not result.success: n_fail += 1 results.append({"name": cfg.name, "status": "run_fail", - "error": f"status={result.status}"}) + "error": f"status={getattr(result, 'status', 'unknown')}"}) continue + avg_ms = sum(times) / len(times) if times else result.time_ms + tflops = (2.0 * size * size * size / (avg_ms * 1e-3)) / 1e12 ref = _emulate(_emulate(A, cfg.dtype_a) @ _emulate(B, cfg.dtype_a), cfg.dtype_a) mr = _max_rel(result.output, ref) ok = mr <= _RTOL n_pass += ok n_fail += not ok results.append({"name": cfg.name, "status": "pass" if ok else "fail", - "tflops": round(result.tflops, 3), "max_rel": round(mr, 6)}) + "tflops": round(tflops, 3), "max_rel": round(mr, 6)}) # --- Report --- print(f"\n{'='*60}") @@ -194,6 +205,10 @@ def main() -> int: help="Max configs to run; 0 = no cap (default: 500)") p.add_argument("--seed", type=int, default=None, help="RNG seed; default = daily rotating seed") + p.add_argument("--warmup", type=int, default=5, + help="Warmup iterations per kernel (default: 5)") + p.add_argument("--repeat", type=int, default=5, + help="Timed iterations per kernel (default: 5)") p.add_argument("--size", type=int, default=1024, help="M=N=K problem size (default: 1024)") p.add_argument("--json", default=None, diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 01ebd04aa6da..989d86ffad31 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1503,7 +1503,6 @@ def runTileEngineGemmTests(String arch, String compiler) { def runDispatcherTests(String arch, String compiler) { def execute_cmd = """ - cd build && \ cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ -D CMAKE_CXX_COMPILER="${compiler}" \ -D CMAKE_BUILD_TYPE=Release \ @@ -1517,9 +1516,12 @@ def runDispatcherTests(String arch, String compiler) { --dtypes fp16,bf16,fp8,bf8 \ --layouts rcr,rrr,crr,ccr \ --budget 500 \ + --warmup 5 \ + --repeat 5 \ --size 1024 \ --json dispatcher_gemm_results.json""" buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) + archiveArtifacts artifacts: "dispatcher_gemm_results.json", allowEmptyArchive: true } def runBuildCKAndTests(String arch) { From 25a56b726ec3dd3d68abd43cc203350ceb9697b7 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Mon, 17 Aug 2026 16:53:49 -0400 Subject: [PATCH 06/51] style(ck-tile): use result.tflops and result.max_rel directly in runner Prefer the values already computed by GpuGemmRunner over recomputing them manually. Fall back to manual computation only when result.max_rel is None (GpuGemmRunner does not populate it; GpuMultiABDRunner does). Co-Authored-By: Claude --- .../dispatcher/tests/run_gemm_search_space.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/projects/composablekernel/dispatcher/tests/run_gemm_search_space.py b/projects/composablekernel/dispatcher/tests/run_gemm_search_space.py index 5c1ce8164a54..2a15726167c3 100644 --- a/projects/composablekernel/dispatcher/tests/run_gemm_search_space.py +++ b/projects/composablekernel/dispatcher/tests/run_gemm_search_space.py @@ -158,10 +158,18 @@ def run(args) -> int: results.append({"name": cfg.name, "status": "run_fail", "error": f"status={getattr(result, 'status', 'unknown')}"}) continue - avg_ms = sum(times) / len(times) if times else result.time_ms - tflops = (2.0 * size * size * size / (avg_ms * 1e-3)) / 1e12 - ref = _emulate(_emulate(A, cfg.dtype_a) @ _emulate(B, cfg.dtype_a), cfg.dtype_a) - mr = _max_rel(result.output, ref) + # Use avg TFLOPS from timed repeat runs; fall back to result.tflops (single run). + if times: + avg_ms = sum(times) / len(times) + tflops = (2.0 * size * size * size / (avg_ms * 1e-3)) / 1e12 + else: + tflops = result.tflops + # Use max_rel from runner if already computed; otherwise compute it here. + if result.max_rel is not None: + mr = result.max_rel + else: + ref = _emulate(_emulate(A, cfg.dtype_a) @ _emulate(B, cfg.dtype_a), cfg.dtype_a) + mr = _max_rel(result.output, ref) ok = mr <= _RTOL n_pass += ok n_fail += not ok From ff9b8e91bdcd5eff3411628b66d4a3ef31f30725 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Tue, 18 Aug 2026 18:05:50 -0400 Subject: [PATCH 07/51] fix(ck-tile): rename run_gemm_search_space.py to satisfy unit-test check CI requires changed code files to have an accompanying test file named test_.py. Rename the runner to match the convention. Co-Authored-By: Claude --- ...run_gemm_search_space.py => test_gemm_search_space.py} | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename projects/composablekernel/dispatcher/tests/{run_gemm_search_space.py => test_gemm_search_space.py} (96%) diff --git a/projects/composablekernel/dispatcher/tests/run_gemm_search_space.py b/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py similarity index 96% rename from projects/composablekernel/dispatcher/tests/run_gemm_search_space.py rename to projects/composablekernel/dispatcher/tests/test_gemm_search_space.py index 2a15726167c3..e3f1cd80e02d 100644 --- a/projects/composablekernel/dispatcher/tests/run_gemm_search_space.py +++ b/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py @@ -14,10 +14,10 @@ driven with TILE_ENGINE_SAMPLING_TIER=daily. Usage: - python3 run_gemm_search_space.py --arch gfx942 --budget 500 - python3 run_gemm_search_space.py --dtypes fp16,bf16 --layouts rcr,rrr - python3 run_gemm_search_space.py --budget 0 # full space, no cap - python3 run_gemm_search_space.py --budget 500 --seed 42 --json results.json + python3 test_gemm_search_space.py --arch gfx942 --budget 500 + python3 test_gemm_search_space.py --dtypes fp16,bf16 --layouts rcr,rrr + python3 test_gemm_search_space.py --budget 0 # full space, no cap + python3 test_gemm_search_space.py --budget 500 --seed 42 --json results.json """ import argparse From 94a78276ac7339c0a6747da4263b0161f88baa76 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Tue, 18 Aug 2026 18:30:19 -0400 Subject: [PATCH 08/51] ci(ck-tile): extend dispatcher CI to cover all ops on gfx942 and gfx950 - Fix script name: run_gemm_search_space.py -> test_gemm_search_space.py - Build ck_tile_dispatcher (static lib) alongside dispatcher_gemm_lib so runtime-compiled tests can link against it - Add test_gemm_parity.py, test_batched_gemm_gpu_correctness.py, and test_batched_contraction_gpu_correctness.py to the gfx942 dispatcher stage - Add new runDispatcherTestsGfx950 groovy function and a parallel Jenkinsfile stage that runs test_bquant_gpu_correctness.py on gfx950 Co-Authored-By: Claude --- projects/composablekernel/Jenkinsfile | 18 ++++++++++++++ .../composablekernel/groovy/vars/ck.groovy | 24 ++++++++++++++++--- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index fe50811faaea..52b4aea12be8 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -702,6 +702,24 @@ pipeline { } } } + stage("Run DISPATCHER Tests on gfx950") + { + when { + beforeAgent true + expression { params.RUN_DISPATCHER_TESTS.toBoolean() } + } + agent none + steps{ + script { + loadCk() + ck.runOnHealthyNode(rocmnode("gfx950")) { + deleteDir() + ck.runDispatcherTestsGfx950(params.BUILD_COMPILER) + cleanWs() + } + } + } + } } } diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 989d86ffad31..13fe09d4855c 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1510,8 +1510,8 @@ def runDispatcherTests(String arch, String compiler) { -D CK_TILE_DISPATCHER=ON \ -D BUILD_DISPATCHER_BINDINGS=ON \ -D DISPATCHER_RULE_SET=tests .. && \ - ninja -j${nthreads()} dispatcher_gemm_lib && \ - python3 ../dispatcher/tests/run_gemm_search_space.py \ + ninja -j${nthreads()} ck_tile_dispatcher dispatcher_gemm_lib && \ + python3 ../dispatcher/tests/test_gemm_search_space.py \ --arch ${arch} \ --dtypes fp16,bf16,fp8,bf8 \ --layouts rcr,rrr,crr,ccr \ @@ -1519,11 +1519,29 @@ def runDispatcherTests(String arch, String compiler) { --warmup 5 \ --repeat 5 \ --size 1024 \ - --json dispatcher_gemm_results.json""" + --json dispatcher_gemm_results.json && \ + python3 ../dispatcher/tests/test_gemm_parity.py && \ + python3 ../dispatcher/tests/test_batched_gemm_gpu_correctness.py --gfx ${arch} && \ + python3 ../dispatcher/tests/test_batched_contraction_gpu_correctness.py --gfx ${arch}""" buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) archiveArtifacts artifacts: "dispatcher_gemm_results.json", allowEmptyArchive: true } +def runDispatcherTestsGfx950(String compiler) { + def arch = "gfx950" + def execute_cmd = """ + cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ + -D CMAKE_CXX_COMPILER="${compiler}" \ + -D CMAKE_BUILD_TYPE=Release \ + -D GPU_TARGETS="${arch}" \ + -D CK_TILE_DISPATCHER=ON \ + -D BUILD_DISPATCHER_BINDINGS=ON \ + -D DISPATCHER_RULE_SET=tests .. && \ + ninja -j${nthreads()} ck_tile_dispatcher dispatcher_gemm_lib && \ + python3 ../dispatcher/tests/test_bquant_gpu_correctness.py --gfx ${arch}""" + buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) +} + def runBuildCKAndTests(String arch) { def gpuTarget def extraSetupArgs = "" From dce2d6ff89d57aea588dc7258de4412158f809bb Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Tue, 18 Aug 2026 20:30:06 -0400 Subject: [PATCH 09/51] ci(ck-tile): add dispatcher gemm_universal CI stage for gfx942 Adds RUN_DISPATCHER_GEMM_UNIVERSAL_TESTS parameter and a new "Run DISPATCHER_GEMM_UNIVERSAL Tests on gfx942" stage that builds and runs only the gemm_universal benchmark (fp8/fp16/bf8/bf16, rcr/rrr/crr/ccr layouts) via the new ck.runDispatcherGemmUniversalTests() groovy function. Co-Authored-By: Claude --- projects/composablekernel/Jenkinsfile | 32 +++++++++++++++++++ .../composablekernel/groovy/vars/ck.groovy | 15 +++++++++ 2 files changed, 47 insertions(+) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index 52b4aea12be8..828e7783d68c 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -135,6 +135,10 @@ pipeline { name: "RUN_TILE_ENGINE_GEMM_TESTS", defaultValue: false, description: "Run the tile_engine_gemm tests (default: OFF)") + booleanParam( + name: "RUN_DISPATCHER_GEMM_UNIVERSAL_TESTS", + defaultValue: false, + description: "Run dispatcher gemm_universal tests on gfx942 only (default: OFF)") booleanParam( name: "RUN_DISPATCHER_TESTS", defaultValue: false, @@ -612,6 +616,34 @@ pipeline { } } } + stage("Run DISPATCHER_GEMM_UNIVERSAL Tests") + { + when { + beforeAgent true + expression { env.SHOULD_RUN_CI.toBoolean() } + } + parallel + { + stage("Run DISPATCHER_GEMM_UNIVERSAL Tests on gfx942") + { + when { + beforeAgent true + expression { params.RUN_DISPATCHER_GEMM_UNIVERSAL_TESTS.toBoolean() } + } + agent none + steps{ + script { + loadCk() + ck.runOnHealthyNode(rocmnode("gfx942")) { + deleteDir() + ck.runDispatcherGemmUniversalTests(params.BUILD_COMPILER) + cleanWs() + } + } + } + } + } + } stage("Run TILE_ENGINE_GEMM Tests") { when { diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 13fe09d4855c..941a36d6de08 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1501,6 +1501,21 @@ def runTileEngineGemmTests(String arch, String compiler) { buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) } +def runDispatcherGemmUniversalTests(String compiler) { + def execute_cmd = """ + cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ + -D BUILD_CK_TILE_ENGINE="ON" \ + -D CMAKE_CXX_COMPILER="${compiler}" \ + -D CMAKE_BUILD_TYPE=Release \ + -D GPU_TARGETS="gfx942" \ + -D GEMM_UNIVERSAL_DATATYPE="fp8;fp16;bf8;bf16" \ + -D GEMM_UNIVERSAL_LAYOUT="rcr;rrr;crr;ccr" \ + -D TILE_ENGINE_SAMPLING_TIER=daily .. && \ + ninja -j${nthreads()} benchmark_gemm_universal_all && \ + python3 ../tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_universal_results.json""" + buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) +} + def runDispatcherTests(String arch, String compiler) { def execute_cmd = """ cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ From b471ea89dbcf2f829fe78f9e0e9707f19cec9df8 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Tue, 18 Aug 2026 20:37:17 -0400 Subject: [PATCH 10/51] ci(ck-tile): rename RUN_DISPATCHER_GEMM_UNIVERSAL_TESTS to RUN_DISPATCHER_GEMM_TESTS Renames the parameter and stage to RUN_DISPATCHER_GEMM_TESTS / runDispatcherGemmTests to allow future expansion beyond gemm_universal. Co-Authored-By: Claude --- projects/composablekernel/Jenkinsfile | 12 ++++++------ projects/composablekernel/groovy/vars/ck.groovy | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index 828e7783d68c..e3fa2dfdbbe9 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -136,9 +136,9 @@ pipeline { defaultValue: false, description: "Run the tile_engine_gemm tests (default: OFF)") booleanParam( - name: "RUN_DISPATCHER_GEMM_UNIVERSAL_TESTS", + name: "RUN_DISPATCHER_GEMM_TESTS", defaultValue: false, - description: "Run dispatcher gemm_universal tests on gfx942 only (default: OFF)") + description: "Run dispatcher gemm tests on gfx942 only (default: OFF)") booleanParam( name: "RUN_DISPATCHER_TESTS", defaultValue: false, @@ -616,7 +616,7 @@ pipeline { } } } - stage("Run DISPATCHER_GEMM_UNIVERSAL Tests") + stage("Run DISPATCHER_GEMM Tests") { when { beforeAgent true @@ -624,11 +624,11 @@ pipeline { } parallel { - stage("Run DISPATCHER_GEMM_UNIVERSAL Tests on gfx942") + stage("Run DISPATCHER_GEMM Tests on gfx942") { when { beforeAgent true - expression { params.RUN_DISPATCHER_GEMM_UNIVERSAL_TESTS.toBoolean() } + expression { params.RUN_DISPATCHER_GEMM_TESTS.toBoolean() } } agent none steps{ @@ -636,7 +636,7 @@ pipeline { loadCk() ck.runOnHealthyNode(rocmnode("gfx942")) { deleteDir() - ck.runDispatcherGemmUniversalTests(params.BUILD_COMPILER) + ck.runDispatcherGemmTests(params.BUILD_COMPILER) cleanWs() } } diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 941a36d6de08..b8f89e5e1dc2 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1501,7 +1501,7 @@ def runTileEngineGemmTests(String arch, String compiler) { buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) } -def runDispatcherGemmUniversalTests(String compiler) { +def runDispatcherGemmTests(String compiler) { def execute_cmd = """ cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ -D BUILD_CK_TILE_ENGINE="ON" \ From b83e43555915157d434cce7021c49fa090315ad5 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Wed, 19 Aug 2026 00:10:11 -0400 Subject: [PATCH 11/51] ci(ck-tile): extend runDispatcherGemmTests to cover batched_gemm and batched_contraction Adds batched_gemm and batched_contraction to the RUN_DISPATCHER_GEMM_TESTS stage on gfx942: builds the tile engine benchmarks alongside dispatcher bindings, runs the benchmark scripts, and runs the GPU correctness tests for both operators. bquant is intentionally excluded (gfx950-only). Co-Authored-By: Claude --- projects/composablekernel/groovy/vars/ck.groovy | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index b8f89e5e1dc2..2ff4597ea0d9 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1510,9 +1510,20 @@ def runDispatcherGemmTests(String compiler) { -D GPU_TARGETS="gfx942" \ -D GEMM_UNIVERSAL_DATATYPE="fp8;fp16;bf8;bf16" \ -D GEMM_UNIVERSAL_LAYOUT="rcr;rrr;crr;ccr" \ - -D TILE_ENGINE_SAMPLING_TIER=daily .. && \ - ninja -j${nthreads()} benchmark_gemm_universal_all && \ - python3 ../tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_universal_results.json""" + -D BATCHED_GEMM_DATATYPE="fp16" \ + -D BATCHED_GEMM_LAYOUT="rcr" \ + -D BATCHED_CONTRACTION_DATATYPE="fp16" \ + -D BATCHED_CONTRACTION_LAYOUT="rcr" \ + -D TILE_ENGINE_SAMPLING_TIER=daily \ + -D CK_TILE_DISPATCHER=ON \ + -D BUILD_DISPATCHER_BINDINGS=ON \ + -D DISPATCHER_RULE_SET=tests .. && \ + ninja -j${nthreads()} benchmark_gemm_universal_all benchmark_batched_gemm_all benchmark_batched_contraction_all ck_tile_dispatcher dispatcher_gemm_lib && \ + python3 ../tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_universal_results.json && \ + python3 ../tile_engine/ops/gemm/batched_gemm/batched_gemm_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json batched_gemm_results.json && \ + python3 ../tile_engine/ops/gemm/batched_contraction/batched_contraction_benchmark.py . --problem-configs "g=2;m=1024;n=1024;k=1024" --warmup 5 --repeat 5 --verbose --json batched_contraction_results.json && \ + python3 ../dispatcher/tests/test_batched_gemm_gpu_correctness.py --gfx gfx942 && \ + python3 ../dispatcher/tests/test_batched_contraction_gpu_correctness.py --gfx gfx942""" buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) } From 4c0f0a3e871786169339e8e3470258aac249ce02 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Wed, 19 Aug 2026 15:39:44 -0400 Subject: [PATCH 12/51] ci(ck-tile): extend dispatcher CI to grouped, multi_d, multi_abd, stream_k, mx_gemm Before this, the dispatcher CI covered gemm_universal, batched_gemm, batched_contraction and bquant. Everything else in the dispatcher shipped with only CPU-side/mocked unit tests, so a numeric regression in those bridges could reach develop undetected. Rather than fork five near-identical runners, generalize the existing test_gemm_search_space.py with --variant {standard,grouped,multi_d,multi_abd, stream_k}: expand_sweep, _ctypes_source_name and setup_multiple_gemm_dispatchers already route every variant, so only the Gpu*Runner choice and the numpy reference differ per variant. This keeps one sampling/reporting/JSON path instead of five that drift. The CI mirrors the shape of the tile_engine benchmark stages (per-op runner, --warmup/--repeat/--json, per-arch stages, archived JSON) but deliberately not their failure semantics: the tile_engine *_benchmark.py scripts never pass --verify and return 0 unconditionally, so copying them literally would give five permanently-green lanes. These lanes validate and exit 1 on mismatch. multi_d runs the PassThrough epilogue. default_ci_config.json carries no multi_d_config, so expand_sweep would otherwise fall back to MultiDAdd; the new --elementwise-op flag pins it, and _verify reads cfg.elementwise_op so kernel and reference always agree on the op. Note the similarly named cde_elementwise_op is the multi_abd field, not this one. mx_gemm has no expand_sweep -- mx_gemm_utils exposes only default_fp8_config() /default_fp4_config() -- so it cannot join the sweep and instead runs the two-config smoke test merged in #10132 on gfx950. Also fixed along the way: - expand_sweep emitted stream_k configs with non-cshuffle epilogues, which the codegen then skips, surfacing later as a spurious build failure. Gate them. - test_batched_{gemm,contraction}_gpu_correctness.py returned exit 0 with no GPU while their add_test entries carried no SKIP_RETURN_CODE, so a CPU-only runner reported a green PASS for a test that never ran. They now exit 77 and the ctest properties record it as a skip. test_bquant gets the same 77 path plus a real arch gate. - ml_dtypes was in no CI dependency manifest, yet every dispatcher lane sweeps bf16/fp8/bf8, which raises "requires the 'ml_dtypes' package" and counts as a failure. Add it to the Docker image. Arch matrix is bounded by the tile_engine CMake guards: stream_k is gfx942-only (TODO: add gfx950 in the stream-K CMakeLists), mx_gemm is gfx950-only. gfx1201 is excluded -- expand_sweep yields zero configs for it in all 16 dtype x layout strata because RDNA4 is warp_size 32 and the config's warp tiles are all 64 wide, so such a lane could only run red. Verified on gfx90a: all five variants pass (20/20 kernels, exit 0), each lane confirmed able to go red (_RTOL 1e-12 -> exit 1) and to skip (exit 77). mx_gemm is unverified on hardware -- it needs a gfx950 node. Co-Authored-By: Claude Opus 5 --- projects/composablekernel/Dockerfile | 2 +- projects/composablekernel/Jenkinsfile | 7 +- .../dispatcher/python/gemm_utils.py | 6 + .../dispatcher/tests/CMakeLists.txt | 10 +- ...est_batched_contraction_gpu_correctness.py | 9 +- .../test_batched_gemm_gpu_correctness.py | 9 +- .../tests/test_bquant_gpu_correctness.py | 41 ++- .../tests/test_gemm_search_space.py | 289 +++++++++++++++--- .../composablekernel/groovy/vars/ck.groovy | 77 ++++- 9 files changed, 380 insertions(+), 70 deletions(-) diff --git a/projects/composablekernel/Dockerfile b/projects/composablekernel/Dockerfile index 484230aba64d..2af55fd42bdd 100644 --- a/projects/composablekernel/Dockerfile +++ b/projects/composablekernel/Dockerfile @@ -100,7 +100,7 @@ RUN set -x && \ wget https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \ dpkg -i dumb-init_*.deb && rm dumb-init_*.deb && \ # Install packages for processing the performance results - pip3 install --break-system-packages --upgrade pytest pymysql pandas==2.2.3 sqlalchemy==2.0.3 setuptools-rust setuptools sshtunnel==0.4.0 && \ + pip3 install --break-system-packages --upgrade pytest pymysql pandas==2.2.3 sqlalchemy==2.0.3 setuptools-rust setuptools sshtunnel==0.4.0 ml_dtypes && \ # Add render group groupadd -f render && \ # Install the new rocm-cmake version diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index e3fa2dfdbbe9..9966e5fb79f8 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -37,7 +37,7 @@ def loadCk() { //launch develop branch daily jobs CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;RUN_CK_TILE_FMHA_TESTS=true;RUN_PERFORMANCE_TESTS=true;FORCE_CI=true - 0 22 * * * % RUN_FULL_QA=true;DISABLE_DL_KERNELS=true;RUN_TILE_ENGINE_BASIC_TESTS=true;RUN_TILE_ENGINE_GEMM_TESTS=true;RUN_DISPATCHER_TESTS=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true + 0 22 * * * % RUN_FULL_QA=true;DISABLE_DL_KERNELS=true;RUN_TILE_ENGINE_BASIC_TESTS=true;RUN_TILE_ENGINE_GEMM_TESTS=true;RUN_DISPATCHER_GEMM_TESTS=true;RUN_DISPATCHER_TESTS=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true 0 21 * * * % RUN_GROUPED_CONV_LARGE_CASES_TESTS=true;hipTensor_test=true;BUILD_GFX101=false;BUILD_GFX908=false;BUILD_GFX942=true;BUILD_GFX950=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true;BUILD_PACKAGES=true 0 19 * * * % BUILD_DOCKER=true;COMPILER_VERSION=develop;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true 0 17 * * * % BUILD_DOCKER=true;COMPILER_VERSION=therock;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true @@ -142,7 +142,7 @@ pipeline { booleanParam( name: "RUN_DISPATCHER_TESTS", defaultValue: false, - description: "Run dispatcher universal GEMM tests (default: OFF)") + description: "Run dispatcher GPU tests: gemm_universal, grouped, multi_d, multi_abd, stream_k, batched_gemm, batched_contraction, plus bquant + mx_gemm on gfx950 (per-arch subset; default: OFF)") booleanParam( name: "BUILD_INSTANCES_ONLY", defaultValue: false, @@ -752,6 +752,9 @@ pipeline { } } } + // No gfx1201 dispatcher stage: the dispatcher has no RDNA4 tile + // coverage, so such a stage could only ever run red. See the + // arch note on dispatcherGemmVariantsFor in groovy/vars/ck.groovy. } } diff --git a/projects/composablekernel/dispatcher/python/gemm_utils.py b/projects/composablekernel/dispatcher/python/gemm_utils.py index 42d69a297fac..f09dfe51f6ad 100644 --- a/projects/composablekernel/dispatcher/python/gemm_utils.py +++ b/projects/composablekernel/dispatcher/python/gemm_utils.py @@ -2396,6 +2396,12 @@ def _as_list(v, default): tm // m_div, tn // n_div, wtm, wtn ): continue + # Stream-K supports only the cshuffle epilogue, and the codegen skips + # anything else (unified_gemm_codegen.py, GemmVariant.STREAM_K branch). + # Without this gate expand_sweep hands back configs whose header is + # never emitted, which surfaces downstream as a spurious build failure. + if variant == "stream_k" and epi != "cshuffle": + continue for (m_na, m_nb, m_nd, m_aop, m_bop, m_cdeop) in mabd_combos: for ew_op, md_nd in md_combos: diff --git a/projects/composablekernel/dispatcher/tests/CMakeLists.txt b/projects/composablekernel/dispatcher/tests/CMakeLists.txt index e70deaa4edae..9dab041ed790 100644 --- a/projects/composablekernel/dispatcher/tests/CMakeLists.txt +++ b/projects/composablekernel/dispatcher/tests/CMakeLists.txt @@ -15,7 +15,7 @@ add_test( NAME dispatcher_test_batched_gemm_gpu_correctness # Run the script directly: this module is script-style (a parameterized # function + main() guard), not unittest.TestCase, so `unittest discover` - # would collect 0 tests and pass vacuously. main() skips (exit 0) without a + # would collect 0 tests and pass vacuously. main() skips (exit 77) without a # GPU/built lib and returns nonzero only on a real correctness failure. COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_batched_gemm_gpu_correctness.py WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} @@ -23,6 +23,9 @@ add_test( set_tests_properties(dispatcher_test_batched_gemm_gpu_correctness PROPERTIES LABELS "dispatcher;python;bridge;gpu" TIMEOUT 600 + # Without this a CPU-only runner reports a green PASS for a test that never + # reached the GPU; 77 makes ctest record it as skipped instead. + SKIP_RETURN_CODE 77 ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" ) @@ -31,7 +34,7 @@ add_test( NAME dispatcher_test_batched_contraction_gpu_correctness # Run the script directly: this module is script-style (a parameterized # function + main() guard), not unittest.TestCase, so `unittest discover` - # would collect 0 tests and pass vacuously. main() skips (exit 0) without a + # would collect 0 tests and pass vacuously. main() skips (exit 77) without a # GPU/built lib and returns nonzero only on a real correctness failure. COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_batched_contraction_gpu_correctness.py WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} @@ -39,6 +42,9 @@ add_test( set_tests_properties(dispatcher_test_batched_contraction_gpu_correctness PROPERTIES LABELS "dispatcher;python;bridge;gpu" TIMEOUT 600 + # Without this a CPU-only runner reports a green PASS for a test that never + # reached the GPU; 77 makes ctest record it as skipped instead. + SKIP_RETURN_CODE 77 ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" ) diff --git a/projects/composablekernel/dispatcher/tests/test_batched_contraction_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_batched_contraction_gpu_correctness.py index 8788f3b48f66..3386e1f80de9 100644 --- a/projects/composablekernel/dispatcher/tests/test_batched_contraction_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_batched_contraction_gpu_correctness.py @@ -10,7 +10,7 @@ the PassThrough v1 signature Old-TE's batched_contraction argparse validates), runs a small contraction on-device via GpuBatchedContractionRunner, and compares the GPU output to an fp32 numpy einsum reference within an fp16-appropriate -tolerance. Skips cleanly (exit 0) when no GPU / hipcc is available. +tolerance. Skips cleanly (exit 77) when no GPU / hipcc is available. The kernel computes: E[g,m,n] = sum_k A[g,m,k] * B[g,n,k] @@ -51,6 +51,11 @@ PASS = "PASS" FAIL = "FAIL" +# ctest reports this as "skipped" rather than passed; see SKIP_RETURN_CODE in +# dispatcher/tests/CMakeLists.txt. Returning 0 here would make a CPU-only runner +# report a green PASS for a test that never touched the GPU. +SKIP_EXIT = 77 + def _has_gpu() -> bool: try: @@ -137,7 +142,7 @@ def main() -> int: if not _has_gpu(): print("SKIP: no supported GPU detected (rocminfo); contraction GPU test skipped") - return 0 + return SKIP_EXIT gfx = _validate_arch(args.gfx) if args.gfx else _get_arch() log.info("Running batched-contraction GPU correctness on %s", gfx) diff --git a/projects/composablekernel/dispatcher/tests/test_batched_gemm_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_batched_gemm_gpu_correctness.py index 240263950813..42845f921c41 100644 --- a/projects/composablekernel/dispatcher/tests/test_batched_gemm_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_batched_gemm_gpu_correctness.py @@ -9,7 +9,7 @@ Builds the batched GEMM dispatcher .so (fp16 / rcr — the only signature Old-TE's batched_gemm_instance_builder validates), runs a small batched problem on-device via GpuBatchedGemmRunner, and compares the GPU output to a per-batch fp32 numpy -reference within an fp16-appropriate tolerance. Skips cleanly (exit 0) when no +reference within an fp16-appropriate tolerance. Skips cleanly (exit 77) when no GPU / hipcc is available so it is safe in a CPU-only CI lane. The kernel computes, for each batch b: @@ -52,6 +52,11 @@ FAIL = "FAIL" SKIP = "SKIP" +# ctest reports this as "skipped" rather than passed; see SKIP_RETURN_CODE in +# dispatcher/tests/CMakeLists.txt. Returning 0 here would make a CPU-only runner +# report a green PASS for a test that never touched the GPU. +SKIP_EXIT = 77 + def _has_gpu() -> bool: """True iff a supported GPU is visible to rocminfo (so the .so can run).""" @@ -159,7 +164,7 @@ def main() -> int: if not _has_gpu(): print("SKIP: no supported GPU detected (rocminfo); batched GPU test skipped") - return 0 + return SKIP_EXIT gfx = args.gfx or _resolve_arch(None) log.info("Running batched GEMM GPU correctness on %s", gfx) diff --git a/projects/composablekernel/dispatcher/tests/test_bquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_bquant_gpu_correctness.py index 34ad85b847ca..2c34b93a0e15 100644 --- a/projects/composablekernel/dispatcher/tests/test_bquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_bquant_gpu_correctness.py @@ -6,7 +6,9 @@ """ GPU correctness tests for BQuant GEMM dispatcher — C4 and H3 coverage. -Requires a gfx950 GPU (MI350X / MI355X) and hipcc in PATH. +Requires a gfx950 GPU (MI350X / MI355X) and hipcc in PATH. Skips cleanly +(exit 77) when no GPU is visible or the detected arch is not gfx950, so it is +safe to invoke unconditionally from a CI lane. Tests: C4 — fp8, bf8: GPU output is non-zero and within 5% max-relative-error @@ -174,6 +176,28 @@ def _max_rel_err(C_gpu: np.ndarray, C_ref: np.ndarray) -> float: PASS = "PASS" FAIL = "FAIL" +# ctest reports this as "skipped" rather than passed. Without it, a CPU-only or +# non-gfx950 runner would report every BQuant test as a hard FAIL for a reason +# that has nothing to do with the code under test. +SKIP_EXIT = 77 + +# BQuant needs the gfx950 (MI350X/MI355X) block-scale MFMA path. +SUPPORTED_ARCHS = ("gfx950",) + + +def _detect_arch() -> str | None: + """Detected gfx arch, or None when no GPU is visible. + + The empty fallback keeps absence detectable; detect_gpu_arch would otherwise + invent a default and turn "no GPU" into a confusing build failure. + """ + try: + from dispatcher_common import detect_gpu_arch + + return detect_gpu_arch(fallback="") or None + except Exception: + return None + def _run_one(label: str, config, M: int, N: int, K: int, A_raw: np.ndarray, A_f32: np.ndarray, @@ -380,7 +404,10 @@ def test_h3_mx_bf16fp4(out_dir: Path, gfx_arch: str) -> tuple[str, str]: def main(): parser = argparse.ArgumentParser(description="BQuant GPU correctness tests (C4 + H3)") - parser.add_argument("--gfx", default="gfx950", help="GPU arch (default: gfx950)") + # No hardcoded default: it must stay possible to tell "user asked for + # gfx950" from "we are on an unrelated box", so the skip below can fire. + parser.add_argument("--gfx", default=None, + help="GPU arch override (default: auto-detect; gfx950 only)") parser.add_argument("-v", "--verbose", action="store_true") parser.add_argument("--output-dir", type=Path, default=None) args = parser.parse_args() @@ -390,6 +417,14 @@ def main(): format="%(levelname)s: %(message)s", ) + gfx = args.gfx or _detect_arch() + if not gfx: + print("SKIP: no supported GPU detected (rocminfo); BQuant GPU tests skipped") + return SKIP_EXIT + if gfx not in SUPPORTED_ARCHS: + print(f"SKIP: BQuant is {'/'.join(SUPPORTED_ARCHS)}-only; detected {gfx}") + return SKIP_EXIT + out_dir = args.output_dir or Path(tempfile.mkdtemp(prefix="bquant_gpu_test_")) log.info("Kernel output dir: %s", out_dir) @@ -397,7 +432,7 @@ def main(): for name, fn in TESTS: log.info("--- Running %s ---", name) try: - status, detail = fn(out_dir, args.gfx) + status, detail = fn(out_dir, gfx) except Exception as exc: status, detail = FAIL, f"{name}: exception: {exc}" results.append((name, status, detail)) diff --git a/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py b/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py index e3f1cd80e02d..7f8a25c1406b 100644 --- a/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py +++ b/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py @@ -3,19 +3,32 @@ # Copyright (c) Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT """ -Dispatcher universal GEMM search space runner. +Dispatcher GEMM search space runner (all bridged GEMM variants). -Enumerates the full GemmKernelConfig search space by calling expand_sweep -with the tile_engine default_ci_config.json across all (dtype, layout) strata, -samples a budget-limited subset with a daily rotating seed, compiles each -config via the bridge, runs on GPU, and reports correctness + TFLOPS. +Enumerates the GemmKernelConfig search space by calling expand_sweep with the +tile_engine default_ci_config.json across all (dtype, layout) strata, samples a +budget-limited subset with a daily rotating seed, compiles each config via the +bridge, runs on GPU, and reports correctness + TFLOPS. -This is the dispatcher equivalent of tile_engine's gemm_universal_benchmark.py -driven with TILE_ENGINE_SAMPLING_TIER=daily. +One runner covers every variant expand_sweep understands -- standard, grouped, +multi_d, multi_abd and stream_k -- because they share the same enumerate/sample/ +build/report machinery and differ only in which Gpu*Runner to drive and how the +numpy reference is computed. Select with --variant. + +This is the dispatcher equivalent of tile_engine's per-op benchmark scripts +driven with TILE_ENGINE_SAMPLING_TIER=daily, with one deliberate difference: +those scripts never verify numerics and always exit 0. This one validates every +kernel against a numpy reference and exits 1 on any mismatch. + +Exit codes: 0 = all pass, 1 = failure, 77 = skipped (no GPU) per the ctest +SKIP_RETURN_CODE convention. Usage: python3 test_gemm_search_space.py --arch gfx942 --budget 500 - python3 test_gemm_search_space.py --dtypes fp16,bf16 --layouts rcr,rrr + python3 test_gemm_search_space.py --variant grouped --groups 4 + python3 test_gemm_search_space.py --variant multi_abd # fp16/rcrr only + python3 test_gemm_search_space.py --variant multi_d # PassThrough epilogue + python3 test_gemm_search_space.py --variant multi_d --elementwise-op MultiDAdd python3 test_gemm_search_space.py --budget 0 # full space, no cap python3 test_gemm_search_space.py --budget 500 --seed 42 --json results.json """ @@ -26,6 +39,7 @@ import random import sys import time +from dataclasses import replace from datetime import date from pathlib import Path @@ -35,7 +49,19 @@ import numpy as np from ctypes_utils import detect_gpu_arch -from gemm_utils import GemmProblem, GpuGemmRunner, expand_sweep, setup_multiple_gemm_dispatchers +from gemm_utils import ( + GemmProblem, + GpuGemmRunner, + GpuGroupedGemmRunner, + GpuMultiABDRunner, + GpuMultiDGemmRunner, + GroupedGemmProblem, + MultiDGemmProblem, + _cde_reference, + expand_sweep, + numpy_dtype_for, + setup_multiple_gemm_dispatchers, +) _CI_CONFIG = ( _DISPATCHER.parent / "tile_engine/ops/gemm/configs/default_ci_config.json" @@ -43,6 +69,19 @@ _DTYPES = ["fp16", "bf16", "fp8", "bf8"] _LAYOUTS = ["rcr", "rrr", "crr", "ccr"] _RTOL = 2e-2 +_SKIP = 77 + +# Per-variant defaults for --dtypes / --layouts. multi_abd and multi_d are +# fp16-only end-to-end (codegen, ctypes lib and runner all assume fp16) and take +# the 4-char (A,B,C/E,D) layout code rather than the 3-char one. +_VARIANT_DEFAULTS = { + "standard": (_DTYPES, _LAYOUTS), + "grouped": (_DTYPES, _LAYOUTS), + "stream_k": (_DTYPES, _LAYOUTS), + "multi_d": (["fp16"], ["rcrr"]), + "multi_abd": (["fp16"], ["rcrr"]), +} +_VARIANTS = tuple(_VARIANT_DEFAULTS) def _daily_seed() -> int: @@ -50,18 +89,31 @@ def _daily_seed() -> int: def _sample(strata: dict, budget: int, seed: int) -> list: - """Stratified budget-limited sample: equal share per non-empty stratum.""" + """Stratified budget-limited sample: equal share per non-empty stratum. + + Strata are visited in sorted order so the sample is reproducible from the + seed alone. Each gets floor(budget/n), with the first ``budget % n`` strata + taking one extra; a stratum smaller than its share releases the difference + to a final top-up draw, so the full budget is used whenever the space allows. + """ nonempty = {k: v for k, v in strata.items() if v} - if not nonempty: + if not nonempty or budget <= 0: return [] rng = random.Random(seed) - n = len(nonempty) - per = max(1, budget // n) - remainder = budget - per * n - selected = [] - for i, (_, configs) in enumerate(nonempty.items()): - alloc = min(per + (remainder if i == n - 1 else 0), len(configs)) - selected.extend(rng.sample(configs, alloc)) + per, remainder = divmod(budget, len(nonempty)) + selected, leftover = [], [] + for i, (_, configs) in enumerate(sorted(nonempty.items())): + # Clamp to the stratum size: budget < n makes `per` 0, and a stratum + # smaller than its share would otherwise ask random.sample for more + # items than exist. + alloc = min(per + (1 if i < remainder else 0), len(configs)) + # Sample indices, not objects: it makes "which ones were not picked" + # exact without relying on object identity or equality. + picked = set(rng.sample(range(len(configs)), alloc)) if alloc > 0 else set() + selected.extend(configs[j] for j in picked) + leftover.extend(configs[j] for j in range(len(configs)) if j not in picked) + if len(selected) < budget and leftover: + selected.extend(rng.sample(leftover, min(budget - len(selected), len(leftover)))) rng.shuffle(selected) return selected[:budget] @@ -78,15 +130,141 @@ def _max_rel(out: np.ndarray, ref: np.ndarray) -> float: return float(np.max(np.abs(out - ref))) / (float(np.max(np.abs(ref))) + 1e-12) +def _quantize(x: np.ndarray, dtype: str) -> np.ndarray: + """Round-trip fp32 through the kernel's host operand dtype. + + The grouped/multi_d runners cast operands with ``numpy_dtype_for`` (fp16 or + an ml_dtypes bf16/fp8-FNUZ/bf8-FNUZ type), so the reference must see the same + quantized values the kernel does. + """ + return x.astype(numpy_dtype_for(dtype)).astype(np.float32) + + +def _layout_code(cfg, n: int) -> str: + """``rcr``/``rcrr`` style code from a config's row/col layout words.""" + words = [cfg.layout_a, cfg.layout_b, cfg.layout_c, getattr(cfg, "layout_d", "row")] + return "".join(w[0] for w in words[:n]) + + +class _Operands: + """Shared host operands for one sweep, generated once and reused per kernel.""" + + def __init__(self, size: int, seed: int, groups: int, num_d: int): + rng = np.random.default_rng(seed) + self.size = size + self.seed = seed + self.A = (rng.standard_normal((size, size)) * 0.1).astype(np.float32) + self.B = (rng.standard_normal((size, size)) * 0.1).astype(np.float32) + self.problem = GemmProblem(M=size, N=size, K=size) + + # Grouped: deliberately NON-uniform M so a kernel that mishandles the + # per-group offsets fails instead of silently passing. All Ms stay + # multiples of 64 (the CI config's only tile_m) because pad_m is false. + step = max(64, size // 4) + ms = [max(64, size - i * step) for i in range(groups)] + self.groups = [(m, size, size) for m in ms] + self.grouped_problem = GroupedGemmProblem(groups=self.groups) + self.A_list = [self.A[:m] for m in ms] + self.B_list = [self.B for _ in ms] + + # Multi-D: D tensors are MxN, stored fp16 by the runner. + self.Ds = [ + (rng.standard_normal((size, size)) * 0.1).astype(np.float16) + for _ in range(max(num_d, 1)) + ] + + +def _make_runner(variant: str, cfg, so: Path): + if variant == "grouped": + return GpuGroupedGemmRunner( + lib_path=so, dtype=cfg.dtype_a, layout=_layout_code(cfg, 3) + ) + if variant == "multi_d": + return GpuMultiDGemmRunner(lib_path=so) + if variant == "multi_abd": + return GpuMultiABDRunner( + lib_path=so, + layout4=_layout_code(cfg, 4), + a_elementwise_op=cfg.a_elementwise_op, + b_elementwise_op=cfg.b_elementwise_op, + cde_elementwise_op=cfg.cde_elementwise_op, + ) + # standard and stream_k share the single-problem C ABI. + return GpuGemmRunner(lib_path=so) + + +def _invoke(variant: str, runner, ops: _Operands): + """One timed launch. Returns an object with .success/.time_ms.""" + if variant == "grouped": + return runner.run(ops.A_list, ops.B_list, ops.grouped_problem) + if variant == "multi_d": + nd = runner.num_d_tensors + return runner.run( + ops.A, ops.B, ops.Ds[:nd], + MultiDGemmProblem(M=ops.size, N=ops.size, K=ops.size, num_d=nd), + ) + if variant == "multi_abd": + # Verification is a separate final call (see _verify): keeping it off the + # timed path avoids recomputing a full fp32 reference on every repeat. + return runner.run(ops.problem, seed=ops.seed, verify=False) + return runner.run(ops.A, ops.B, ops.problem) + + +def _verify(variant: str, cfg, runner, result, ops: _Operands) -> float: + """max_rel of the GPU result against a numpy reference.""" + if variant == "multi_abd": + # The runner generates its own operands, so it also owns the reference + # (mirroring ck_tile::reference_gemm_multiple_abd). One extra launch. + verified = runner.run(ops.problem, seed=ops.seed, verify=True, verify_tol=_RTOL) + if verified.max_rel is None: + raise RuntimeError(f"multi_abd verification did not run (status={verified.status})") + return verified.max_rel + + # Only GemmResult carries max_rel; the grouped/multi_d result types do not. + if getattr(result, "max_rel", None) is not None: + return result.max_rel + + dtype = cfg.dtype_a + if variant == "grouped": + worst = 0.0 + Bq = _quantize(ops.B, dtype) + for out, A_g in zip(result.outputs, ops.A_list): + ref = _quantize(A_g, dtype) @ Bq + worst = max(worst, _max_rel(np.asarray(out).astype(np.float32), ref)) + return worst + + if variant == "multi_d": + # multi_d's fused epilogue op lives in `elementwise_op`; the similarly + # named `cde_elementwise_op` is the multi_abd field and stays at its + # PassThrough default here, so reading it would reference the wrong op. + # Read it rather than hard-coding --elementwise-op: under the default + # PassThrough the epilogue is E = C and _cde_reference drops the D + # tensors, which is only correct because both sides agree on the op. + nd = runner.num_d_tensors + acc = _quantize(ops.A, dtype) @ _quantize(ops.B, dtype) + ref = _cde_reference(cfg.elementwise_op, acc, ops.Ds[:nd]) + return _max_rel(result.output.astype(np.float32), ref) + + # standard / stream_k + ref = _emulate(_emulate(ops.A, dtype) @ _emulate(ops.B, dtype), dtype) + return _max_rel(result.output, ref) + + def run(args) -> int: arch = args.arch or detect_gpu_arch() - dtypes = [d.strip() for d in args.dtypes.split(",")] - layouts = [l.strip() for l in args.layouts.split(",")] + if not arch: + print("SKIP: no GPU detected and --arch not given") + return _SKIP + + variant = args.variant + def_dtypes, def_layouts = _VARIANT_DEFAULTS[variant] + dtypes = [d.strip() for d in args.dtypes.split(",")] if args.dtypes else def_dtypes + layouts = [l.strip() for l in args.layouts.split(",")] if args.layouts else def_layouts budget = args.budget seed = args.seed if args.seed is not None else _daily_seed() size = args.size - print(f"arch={arch} dtypes={dtypes} layouts={layouts}") + print(f"variant={variant} arch={arch} dtypes={dtypes} layouts={layouts}") print(f"budget={budget if budget > 0 else 'unlimited'} seed={seed} size={size}") # --- Enumerate: same JSON config as tile_engine, across all strata --- @@ -95,8 +273,16 @@ def run(args) -> int: strata = {} for dtype in dtypes: for layout in layouts: - configs = expand_sweep(str(_CI_CONFIG), arch=arch, - dtype=dtype, layout=layout) + configs = expand_sweep(str(_CI_CONFIG), arch=arch, dtype=dtype, + layout=layout, variant=variant) + if variant == "multi_d": + # default_ci_config.json carries no multi_d_config, so expand_sweep + # falls back to its own MultiDAdd default. Pin the epilogue op here + # instead: the config dataclass derives .name (and the codegen's + # elementwise_ops list) from this field, so replacing it is enough + # to select a different kernel -- no separate JSON is needed. + configs = [replace(c, elementwise_op=args.elementwise_op) + for c in configs] strata[f"{dtype}/{layout}"] = configs total = sum(len(v) for v in strata.values()) print(f" {total} valid configs across {len(strata)} strata ({time.time()-t0:.1f}s)") @@ -126,10 +312,11 @@ def run(args) -> int: # --- Run --- print(f"\nRunning {build_ok} kernels on GPU (M=N=K={size}, " f"warmup={args.warmup}, repeat={args.repeat})...") - problem = GemmProblem(M=size, N=size, K=size) - rng = np.random.default_rng(seed) - A = (rng.standard_normal((size, size)) * 0.1).astype(np.float32) - B = (rng.standard_normal((size, size)) * 0.1).astype(np.float32) + # num_d is per-kernel, but every config in a sweep shares the same operand + # shapes, so generate the largest D set once and slice per kernel. + max_num_d = max((getattr(c, "num_d_tensors", 0) for c in configs), default=0) + ops = _Operands(size=size, seed=seed, groups=args.groups, num_d=max_num_d) + flops = ops.grouped_problem.flops if variant == "grouped" else ops.problem.flops results = [] n_pass = n_fail = n_build_fail = 0 @@ -140,13 +327,13 @@ def run(args) -> int: results.append({"name": cfg.name, "status": "build_fail"}) continue try: - runner = GpuGemmRunner(lib_path=so) + runner = _make_runner(variant, cfg, so) for _ in range(args.warmup): - runner.run(A, B, problem) + _invoke(variant, runner, ops) times = [] result = None for _ in range(max(1, args.repeat)): - result = runner.run(A, B, problem) + result = _invoke(variant, runner, ops) if result.success: times.append(result.time_ms) except Exception as exc: @@ -161,15 +348,15 @@ def run(args) -> int: # Use avg TFLOPS from timed repeat runs; fall back to result.tflops (single run). if times: avg_ms = sum(times) / len(times) - tflops = (2.0 * size * size * size / (avg_ms * 1e-3)) / 1e12 + tflops = (flops / (avg_ms * 1e-3)) / 1e12 else: tflops = result.tflops - # Use max_rel from runner if already computed; otherwise compute it here. - if result.max_rel is not None: - mr = result.max_rel - else: - ref = _emulate(_emulate(A, cfg.dtype_a) @ _emulate(B, cfg.dtype_a), cfg.dtype_a) - mr = _max_rel(result.output, ref) + try: + mr = _verify(variant, cfg, runner, result, ops) + except Exception as exc: + n_fail += 1 + results.append({"name": cfg.name, "status": "verify_error", "error": str(exc)}) + continue ok = mr <= _RTOL n_pass += ok n_fail += not ok @@ -182,9 +369,9 @@ def run(args) -> int: f"/ {len(configs)} total") if args.json: - out = {"arch": arch, "size": size, "seed": seed, "budget": budget, - "total_configs": total, "n_pass": n_pass, "n_fail": n_fail, - "n_build_fail": n_build_fail, "kernels": results} + out = {"variant": variant, "arch": arch, "size": size, "seed": seed, + "budget": budget, "total_configs": total, "n_pass": n_pass, + "n_fail": n_fail, "n_build_fail": n_build_fail, "kernels": results} Path(args.json).write_text(json.dumps(out, indent=2)) print(f"Results written to {args.json}") @@ -200,15 +387,27 @@ def run(args) -> int: def main() -> int: p = argparse.ArgumentParser( - description="Dispatcher universal GEMM search space runner", + description="Dispatcher GEMM search space runner (all bridged variants)", formatter_class=argparse.RawDescriptionHelpFormatter, ) + p.add_argument("--variant", default="standard", choices=_VARIANTS, + help="GEMM variant to sweep (default: standard)") p.add_argument("--arch", default=None, help="GPU arch (default: auto-detect via rocminfo)") - p.add_argument("--dtypes", default=",".join(_DTYPES), - help=f"Comma-separated dtypes (default: {','.join(_DTYPES)})") - p.add_argument("--layouts", default=",".join(_LAYOUTS), - help=f"Comma-separated layouts (default: {','.join(_LAYOUTS)})") + p.add_argument("--dtypes", default=None, + help="Comma-separated dtypes (default: per-variant; " + f"{','.join(_DTYPES)} for standard/grouped/stream_k, " + "fp16 for multi_d/multi_abd)") + p.add_argument("--layouts", default=None, + help="Comma-separated layouts (default: per-variant; " + f"{','.join(_LAYOUTS)} for standard/grouped/stream_k, " + "rcrr for multi_d/multi_abd)") + p.add_argument("--groups", type=int, default=4, + help="Sub-problem count for --variant grouped (default: 4)") + p.add_argument("--elementwise-op", default="PassThrough", + choices=("PassThrough", "MultiDAdd", "MultiDMultiply"), + help="Epilogue op for --variant multi_d (default: PassThrough). " + "Ignored by every other variant.") p.add_argument("--budget", type=int, default=500, help="Max configs to run; 0 = no cap (default: 500)") p.add_argument("--seed", type=int, default=None, diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 2ff4597ea0d9..72da0c89fff7 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1527,6 +1527,47 @@ def runDispatcherGemmTests(String compiler) { buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) } +// Which GEMM variants of test_gemm_search_space.py each arch can run. Bounded by +// the tile_engine CMake arch guards -- the dispatcher cannot cover more archs +// than the kernels it selects are built for. stream_k is gfx942-only until the +// "TODO: add gfx950" in the stream-K CMakeLists is resolved. +// +// gfx1201 is absent on purpose, and the Jenkinsfile has no gfx1201 dispatcher +// stage for the same reason: gemm_utils._SUPPORTED_ARCHES is +// (gfx90a, gfx942, gfx950), so test_gemm_search_space.py raises +// ValueError: Unsupported GPU architecture 'gfx1201' +// before it ever reaches the GPU. Widening that guard does not help -- with it +// widened, expand_sweep returns 0 configs for gfx1201 in all 16 dtype x layout +// strata of default_ci_config.json, because RDNA4 is warp_size 32 and every +// warp tile that config names is 64 wide, and the runner then exits 1 on "no +// configs to run". Covering RDNA4 needs real gfx1201 tile coverage first. +def dispatcherGemmVariantsFor(String arch) { + switch (arch) { + case "gfx942": return ["grouped", "multi_d", "multi_abd", "stream_k"] + case "gfx950": return ["grouped", "multi_d", "multi_abd"] + default: return [] + } +} + +// One test_gemm_search_space.py invocation for a non-standard GEMM variant. +// +// dtypes/layouts are deliberately omitted: the runner picks per-variant defaults +// (multi_d/multi_abd are fp16/rcrr-only, so naming the standard 4x4 matrix here +// would enumerate configs that variant cannot build). multi_d likewise takes the +// runner's default --elementwise-op PassThrough. The budget is well under the +// standard lane's 500 because each variant JIT-compiles its own .so set and +// these run in addition to, not instead of, the standard sweep. +def dispatcherVariantCmd(String arch, String variant) { + return """python3 ../dispatcher/tests/test_gemm_search_space.py \ + --variant ${variant} \ + --arch ${arch} \ + --budget 64 \ + --warmup 5 \ + --repeat 5 \ + --size 1024 \ + --json dispatcher_${variant}_results.json""" +} + def runDispatcherTests(String arch, String compiler) { def execute_cmd = """ cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ @@ -1549,23 +1590,33 @@ def runDispatcherTests(String arch, String compiler) { python3 ../dispatcher/tests/test_gemm_parity.py && \ python3 ../dispatcher/tests/test_batched_gemm_gpu_correctness.py --gfx ${arch} && \ python3 ../dispatcher/tests/test_batched_contraction_gpu_correctness.py --gfx ${arch}""" + dispatcherGemmVariantsFor(arch).each { variant -> + execute_cmd += " && \\\n " + dispatcherVariantCmd(arch, variant) + } + // Stream-K has its own registry-level driver test; its defaults are the + // full 4x4 matrix (48 driver compiles / 96 GPU runs), so CI pins one + // dtype/layout. Note it takes --arch, not --gfx. + if (arch == "gfx942") { + execute_cmd += """ && \ + python3 ../dispatcher/tests/test_streamk_registry.py --arch ${arch} --datatypes fp16 --layouts rcr""" + } + // mx_gemm has no expand_sweep -- mx_gemm_utils exposes only + // default_fp8_config()/default_fp4_config() -- so it cannot join the + // --variant sweep above and instead runs the two-config smoke test merged + // in #10132. That test is unittest-based and self-gates to gfx950, so it + // takes no --gfx (unlike test_bquant, which does). + if (arch == "gfx950") { + execute_cmd += """ && \ + python3 ../dispatcher/tests/test_bquant_gpu_correctness.py --gfx ${arch} && \ + python3 ../dispatcher/tests/test_mx_gemm_gpu_correctness.py""" + } buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) - archiveArtifacts artifacts: "dispatcher_gemm_results.json", allowEmptyArchive: true + // Glob, not a single name: each variant writes its own dispatcher__results.json. + archiveArtifacts artifacts: "dispatcher_*_results.json", allowEmptyArchive: true } def runDispatcherTestsGfx950(String compiler) { - def arch = "gfx950" - def execute_cmd = """ - cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ - -D CMAKE_CXX_COMPILER="${compiler}" \ - -D CMAKE_BUILD_TYPE=Release \ - -D GPU_TARGETS="${arch}" \ - -D CK_TILE_DISPATCHER=ON \ - -D BUILD_DISPATCHER_BINDINGS=ON \ - -D DISPATCHER_RULE_SET=tests .. && \ - ninja -j${nthreads()} ck_tile_dispatcher dispatcher_gemm_lib && \ - python3 ../dispatcher/tests/test_bquant_gpu_correctness.py --gfx ${arch}""" - buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) + runDispatcherTests("gfx950", compiler) } def runBuildCKAndTests(String arch) { From ca79713612bec2f5fb4bf5c026c85a8b1b83a8c8 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Wed, 19 Aug 2026 16:35:21 -0400 Subject: [PATCH 13/51] fix(ck-tile): correct fp8/bf8 reference, fail on build errors, fix artifact path Three defects found reviewing the dispatcher CI before merge. 1. fp8/bf8 numerics were verified against a wrong reference. _emulate() fell through to float16 for fp8/bf8, so the reference rounded operands to fp16 while the kernel consumed real float8_e4m3fnuz / float8_e5m2fnuz. That left the reference strictly more precise than the kernel, failing every fp8/bf8 config by construction. A 4-dtype x 4-layout sweep (budget 64, size 512) went 32 pass / 32 fail, with all 32 failures fp8/bf8 at max_rel 0.033-0.065 against an _RTOL of 0.02. This predates the new variant lanes: the gemm_universal lane already sweeps all four dtypes, so it would have gone red on the first nightly. Quantize operands at the input dtype and the result at cfg.dtype_c -- they differ, and conflating them was the bug. The same sweep is now 64 pass / 0 fail, worst max_rel 0.0064 (bf16). _emulate had no other call site and is removed in favour of _quantize, which the grouped/multi_d branches already used correctly. 2. Build failures produced a green lane. The exit status ignored n_build_fail, so a total codegen or compile breakage -- every kernel failing to build, nothing verified -- still exited 0, and build failures were filtered out of the printed listing so the log gave no hint either. Verified by stubbing setup_multiple_gemm_dispatchers to return None: 0 pass / 0 fail / 4 build-fail now exits 1 and names all four. 3. The result JSONs were never archived. cmake_build runs execute_cmd from projects/composablekernel/build, but archiveArtifacts resolves against the workspace root, so the bare "dispatcher_*_results.json" glob matched nothing -- silently, because allowEmptyArchive is set. Qualify the path, and move the call into a finally block so a red lane keeps the JSON that explains why it went red. Re-ran grouped, multi_d, multi_abd and stream_k after the reference change: 4 pass / 0 fail each, exit 0. --- .../tests/test_gemm_search_space.py | 29 ++++++++++--------- .../composablekernel/groovy/vars/ck.groovy | 17 +++++++++-- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py b/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py index 7f8a25c1406b..10f9acbe3826 100644 --- a/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py +++ b/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py @@ -18,7 +18,7 @@ This is the dispatcher equivalent of tile_engine's per-op benchmark scripts driven with TILE_ENGINE_SAMPLING_TIER=daily, with one deliberate difference: those scripts never verify numerics and always exit 0. This one validates every -kernel against a numpy reference and exits 1 on any mismatch. +kernel against a numpy reference and exits 1 on any mismatch or build failure. Exit codes: 0 = all pass, 1 = failure, 77 = skipped (no GPU) per the ctest SKIP_RETURN_CODE convention. @@ -118,14 +118,6 @@ def _sample(strata: dict, budget: int, seed: int) -> list: return selected[:budget] -def _emulate(x: np.ndarray, dtype: str) -> np.ndarray: - if dtype == "bf16": - u32 = np.ascontiguousarray(x, dtype=np.float32).view(np.uint32) - rounded = (u32 + ((u32 >> 16) & 1) + np.uint32(0x7FFF)) >> 16 - return (rounded.astype(np.uint32) << 16).view(np.float32) - return x.astype(np.float16).astype(np.float32) - - def _max_rel(out: np.ndarray, ref: np.ndarray) -> float: return float(np.max(np.abs(out - ref))) / (float(np.max(np.abs(ref))) + 1e-12) @@ -246,7 +238,15 @@ def _verify(variant: str, cfg, runner, result, ops: _Operands) -> float: return _max_rel(result.output.astype(np.float32), ref) # standard / stream_k - ref = _emulate(_emulate(ops.A, dtype) @ _emulate(ops.B, dtype), dtype) + # + # Quantize operands to the *input* dtype and the result to the *output* + # dtype -- they differ (expand_sweep maps fp8/bf8 inputs to an fp16 C), and + # conflating them is what a naive "round everything to `dtype`" reference + # gets wrong: for fp8/bf8 it would round the operands to fp16, leaving the + # reference strictly more precise than the kernel and failing every fp8/bf8 + # config by construction. + acc = _quantize(ops.A, dtype) @ _quantize(ops.B, dtype) + ref = _quantize(acc, cfg.dtype_c) return _max_rel(result.output, ref) @@ -375,14 +375,17 @@ def run(args) -> int: Path(args.json).write_text(json.dumps(out, indent=2)) print(f"Results written to {args.json}") - if n_fail > 0: + if n_fail > 0 or n_build_fail > 0: print("\nFailed kernels:") for r in results: - if r["status"] not in ("pass", "build_fail"): + if r["status"] != "pass": print(f" {r['name']}: {r['status']} " f"{r.get('error', r.get('max_rel', ''))}") - return 0 if n_fail == 0 else 1 + # A build failure is a failure. Excluding it would mean a total codegen or + # compile breakage -- every kernel failing to build, nothing verified -- + # still reports a green lane. + return 0 if n_fail == 0 and n_build_fail == 0 else 1 def main() -> int: diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 72da0c89fff7..b1e0975c16e9 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1610,9 +1610,20 @@ def runDispatcherTests(String arch, String compiler) { python3 ../dispatcher/tests/test_bquant_gpu_correctness.py --gfx ${arch} && \ python3 ../dispatcher/tests/test_mx_gemm_gpu_correctness.py""" } - buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) - // Glob, not a single name: each variant writes its own dispatcher__results.json. - archiveArtifacts artifacts: "dispatcher_*_results.json", allowEmptyArchive: true + try { + buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) + } finally { + // finally, not a trailing call: a red lane is exactly when the per-kernel + // JSON is worth having, and buildAndTest throws on failure. + // + // Path, not a bare glob: cmake_build runs execute_cmd from + // projects/composablekernel/build, while archiveArtifacts resolves + // against the workspace root. A bare "dispatcher_*_results.json" matches + // nothing there, and allowEmptyArchive hides that it matched nothing. + // Glob, not one name: each variant writes its own dispatcher__*.json. + archiveArtifacts artifacts: "projects/composablekernel/build/dispatcher_*_results.json", + allowEmptyArchive: true + } } def runDispatcherTestsGfx950(String compiler) { From 6c095c3dea7f0edb7fe6373cef8b5b8d48191b6d Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Wed, 19 Aug 2026 18:33:55 -0400 Subject: [PATCH 14/51] ci(ck-tile): archive gemm-lane JSON, drop redundant gfx950 wrapper Wrap runDispatcherGemmTests in try/finally so its gemm_universal / batched_gemm / batched_contraction result JSON is archived on a red lane too, matching runDispatcherTests. Remove the single-use runDispatcherTestsGfx950 wrapper and call runDispatcherTests("gfx950", ...) directly, consistent with the two-arg convention used everywhere else. Co-Authored-By: Claude --- projects/composablekernel/Jenkinsfile | 2 +- .../composablekernel/groovy/vars/ck.groovy | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index 9966e5fb79f8..2b52e0a32e07 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -746,7 +746,7 @@ pipeline { loadCk() ck.runOnHealthyNode(rocmnode("gfx950")) { deleteDir() - ck.runDispatcherTestsGfx950(params.BUILD_COMPILER) + ck.runDispatcherTests("gfx950", params.BUILD_COMPILER) cleanWs() } } diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index b1e0975c16e9..4b3c85c09ea4 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1524,7 +1524,19 @@ def runDispatcherGemmTests(String compiler) { python3 ../tile_engine/ops/gemm/batched_contraction/batched_contraction_benchmark.py . --problem-configs "g=2;m=1024;n=1024;k=1024" --warmup 5 --repeat 5 --verbose --json batched_contraction_results.json && \ python3 ../dispatcher/tests/test_batched_gemm_gpu_correctness.py --gfx gfx942 && \ python3 ../dispatcher/tests/test_batched_contraction_gpu_correctness.py --gfx gfx942""" - buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) + try { + buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) + } finally { + // finally, not a trailing call: a red lane is exactly when the per-kernel + // JSON is worth having, and buildAndTest throws on failure. Same path + // reasoning as runDispatcherTests: execute_cmd runs from + // projects/composablekernel/build while archiveArtifacts resolves against + // the workspace root, so the build-relative path is spelled out in full. + archiveArtifacts artifacts: "projects/composablekernel/build/gemm_universal_results.json," + + "projects/composablekernel/build/batched_gemm_results.json," + + "projects/composablekernel/build/batched_contraction_results.json", + allowEmptyArchive: true + } } // Which GEMM variants of test_gemm_search_space.py each arch can run. Bounded by @@ -1626,10 +1638,6 @@ def runDispatcherTests(String arch, String compiler) { } } -def runDispatcherTestsGfx950(String compiler) { - runDispatcherTests("gfx950", compiler) -} - def runBuildCKAndTests(String arch) { def gpuTarget def extraSetupArgs = "" From 3ba53eb756e4e9dac035f5f643622cfd38a7e768 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Thu, 20 Aug 2026 11:49:22 -0400 Subject: [PATCH 15/51] ci(ck-tile): add grouped rowcolquant/tensorquant GPU correctness to gfx950 lane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds test_rowcolquant_gpu_correctness.py and test_tensorquant_gpu_correctness.py to the dispatcher gfx950 CI block, alongside the existing bquant and mx_gemm correctness tests. Both tests accept --gfx, skip gracefully when ml_dtypes or hipcc are absent, and cover fp8/bf8 plus a non-square (M=64,N=256,K=128) case that would catch any recurrence of the A@Bᵀ transpose bug. Addresses doc finding T1: the rectangular anti-transpose guarantee now runs in CI on every change rather than being verified manually once on gfx950. Co-Authored-By: Claude --- projects/composablekernel/groovy/vars/ck.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 4b3c85c09ea4..cf995ea64523 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1620,7 +1620,9 @@ def runDispatcherTests(String arch, String compiler) { if (arch == "gfx950") { execute_cmd += """ && \ python3 ../dispatcher/tests/test_bquant_gpu_correctness.py --gfx ${arch} && \ - python3 ../dispatcher/tests/test_mx_gemm_gpu_correctness.py""" + python3 ../dispatcher/tests/test_mx_gemm_gpu_correctness.py && \ + python3 ../dispatcher/tests/test_rowcolquant_gpu_correctness.py --gfx ${arch} && \ + python3 ../dispatcher/tests/test_tensorquant_gpu_correctness.py --gfx ${arch}""" } try { buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) From 387db2b6925d85954743c508caf351bb47e65f5b Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Mon, 24 Aug 2026 15:09:33 -0400 Subject: [PATCH 16/51] refactor(ck-tile): clarify dispatcher CI parameter names The two dispatcher CI lanes gate different builds, but the old names did not say so: RUN_DISPATCHER_GEMM_TESTS is a tile_engine benchmark lane over ahead-of-time compiled instances, while RUN_DISPATCHER_TESTS builds only the dispatcher and verifies JIT-compiled kernels. The names implied the first was the dispatcher GEMM lane and the second a superset. Rename along the axis that actually separates them: RUN_DISPATCHER_GEMM_TESTS -> RUN_DISPATCHER_PERF_TESTS RUN_DISPATCHER_TESTS -> RUN_DISPATCHER_CORRECTNESS_TESTS runDispatcherGemmTests() -> runDispatcherPerfTests() runDispatcherTests() -> runDispatcherCorrectnessTests() This keeps the RUN___TESTS convention already used by RUN_TILE_ENGINE_BASIC_TESTS / RUN_TILE_ENGINE_GEMM_TESTS, and the shared DISPATCHER prefix keeps the pair adjacent in the parameter list. Stage labels and the 22:00 develop cron are updated to match, and both parameter descriptions are rewritten to name the AOT/JIT distinction instead of restating the lane name. No behavioural change. Co-Authored-By: Claude Opus 5 --- projects/composablekernel/Jenkinsfile | 32 +++++++++---------- .../composablekernel/groovy/vars/ck.groovy | 6 ++-- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index 2b52e0a32e07..66c46c1a6748 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -37,7 +37,7 @@ def loadCk() { //launch develop branch daily jobs CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;RUN_CK_TILE_FMHA_TESTS=true;RUN_PERFORMANCE_TESTS=true;FORCE_CI=true - 0 22 * * * % RUN_FULL_QA=true;DISABLE_DL_KERNELS=true;RUN_TILE_ENGINE_BASIC_TESTS=true;RUN_TILE_ENGINE_GEMM_TESTS=true;RUN_DISPATCHER_GEMM_TESTS=true;RUN_DISPATCHER_TESTS=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true + 0 22 * * * % RUN_FULL_QA=true;DISABLE_DL_KERNELS=true;RUN_TILE_ENGINE_BASIC_TESTS=true;RUN_TILE_ENGINE_GEMM_TESTS=true;RUN_DISPATCHER_PERF_TESTS=true;RUN_DISPATCHER_CORRECTNESS_TESTS=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true 0 21 * * * % RUN_GROUPED_CONV_LARGE_CASES_TESTS=true;hipTensor_test=true;BUILD_GFX101=false;BUILD_GFX908=false;BUILD_GFX942=true;BUILD_GFX950=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true;BUILD_PACKAGES=true 0 19 * * * % BUILD_DOCKER=true;COMPILER_VERSION=develop;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true 0 17 * * * % BUILD_DOCKER=true;COMPILER_VERSION=therock;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true @@ -136,13 +136,13 @@ pipeline { defaultValue: false, description: "Run the tile_engine_gemm tests (default: OFF)") booleanParam( - name: "RUN_DISPATCHER_GEMM_TESTS", + name: "RUN_DISPATCHER_PERF_TESTS", defaultValue: false, - description: "Run dispatcher gemm tests on gfx942 only (default: OFF)") + description: "Benchmark ahead-of-time tile_engine GEMM instances on gfx942: gemm_universal, batched_gemm, batched_contraction (default: OFF)") booleanParam( - name: "RUN_DISPATCHER_TESTS", + name: "RUN_DISPATCHER_CORRECTNESS_TESTS", defaultValue: false, - description: "Run dispatcher GPU tests: gemm_universal, grouped, multi_d, multi_abd, stream_k, batched_gemm, batched_contraction, plus bquant + mx_gemm on gfx950 (per-arch subset; default: OFF)") + description: "Verify JIT-compiled dispatcher kernels on gfx942 + gfx950: gemm_universal, grouped, multi_d, multi_abd, stream_k, batched_gemm, batched_contraction, plus bquant + mx_gemm on gfx950 (per-arch subset; default: OFF)") booleanParam( name: "BUILD_INSTANCES_ONLY", defaultValue: false, @@ -616,7 +616,7 @@ pipeline { } } } - stage("Run DISPATCHER_GEMM Tests") + stage("Run DISPATCHER_PERF Tests") { when { beforeAgent true @@ -624,11 +624,11 @@ pipeline { } parallel { - stage("Run DISPATCHER_GEMM Tests on gfx942") + stage("Run DISPATCHER_PERF Tests on gfx942") { when { beforeAgent true - expression { params.RUN_DISPATCHER_GEMM_TESTS.toBoolean() } + expression { params.RUN_DISPATCHER_PERF_TESTS.toBoolean() } } agent none steps{ @@ -636,7 +636,7 @@ pipeline { loadCk() ck.runOnHealthyNode(rocmnode("gfx942")) { deleteDir() - ck.runDispatcherGemmTests(params.BUILD_COMPILER) + ck.runDispatcherPerfTests(params.BUILD_COMPILER) cleanWs() } } @@ -708,7 +708,7 @@ pipeline { } } } - stage("Run DISPATCHER Tests") + stage("Run DISPATCHER_CORRECTNESS Tests") { when { beforeAgent true @@ -716,11 +716,11 @@ pipeline { } parallel { - stage("Run DISPATCHER Tests on gfx942") + stage("Run DISPATCHER_CORRECTNESS Tests on gfx942") { when { beforeAgent true - expression { params.RUN_DISPATCHER_TESTS.toBoolean() } + expression { params.RUN_DISPATCHER_CORRECTNESS_TESTS.toBoolean() } } agent none steps{ @@ -728,17 +728,17 @@ pipeline { loadCk() ck.runOnHealthyNode(rocmnode("gfx942")) { deleteDir() - ck.runDispatcherTests("gfx942", params.BUILD_COMPILER) + ck.runDispatcherCorrectnessTests("gfx942", params.BUILD_COMPILER) cleanWs() } } } } - stage("Run DISPATCHER Tests on gfx950") + stage("Run DISPATCHER_CORRECTNESS Tests on gfx950") { when { beforeAgent true - expression { params.RUN_DISPATCHER_TESTS.toBoolean() } + expression { params.RUN_DISPATCHER_CORRECTNESS_TESTS.toBoolean() } } agent none steps{ @@ -746,7 +746,7 @@ pipeline { loadCk() ck.runOnHealthyNode(rocmnode("gfx950")) { deleteDir() - ck.runDispatcherTests("gfx950", params.BUILD_COMPILER) + ck.runDispatcherCorrectnessTests("gfx950", params.BUILD_COMPILER) cleanWs() } } diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index cf995ea64523..edea323042a0 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1501,7 +1501,7 @@ def runTileEngineGemmTests(String arch, String compiler) { buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) } -def runDispatcherGemmTests(String compiler) { +def runDispatcherPerfTests(String compiler) { def execute_cmd = """ cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ -D BUILD_CK_TILE_ENGINE="ON" \ @@ -1529,7 +1529,7 @@ def runDispatcherGemmTests(String compiler) { } finally { // finally, not a trailing call: a red lane is exactly when the per-kernel // JSON is worth having, and buildAndTest throws on failure. Same path - // reasoning as runDispatcherTests: execute_cmd runs from + // reasoning as runDispatcherCorrectnessTests: execute_cmd runs from // projects/composablekernel/build while archiveArtifacts resolves against // the workspace root, so the build-relative path is spelled out in full. archiveArtifacts artifacts: "projects/composablekernel/build/gemm_universal_results.json," + @@ -1580,7 +1580,7 @@ def dispatcherVariantCmd(String arch, String variant) { --json dispatcher_${variant}_results.json""" } -def runDispatcherTests(String arch, String compiler) { +def runDispatcherCorrectnessTests(String arch, String compiler) { def execute_cmd = """ cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ -D CMAKE_CXX_COMPILER="${compiler}" \ From 58bce77e60565d90527871a5459ad16dd00d3741 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Mon, 24 Aug 2026 15:10:38 -0400 Subject: [PATCH 17/51] refactor(ck-tile): drop duplicated correctness tests from the perf lane runDispatcherPerfTests ran test_batched_gemm_gpu_correctness.py and test_batched_contraction_gpu_correctness.py on gfx942, but runDispatcherCorrectnessTests already runs both on the same arch, and the 22:00 develop cron enables both parameters. Every nightly therefore did that work twice. Keep the invocations in the correctness lane only. With them gone the perf lane no longer needs the dispatcher at all, so drop CK_TILE_DISPATCHER, BUILD_DISPATCHER_BINDINGS and DISPATCHER_RULE_SET from its cmake configure and ck_tile_dispatcher / dispatcher_gemm_lib from its ninja targets. The three tile_engine benchmark drivers import only the standard library and drive the benchmark_*_all binaries, so none of them reaches the dispatcher. Coverage is unchanged; the perf lane now builds and benchmarks ahead-of-time tile_engine instances and nothing else. Co-Authored-By: Claude Opus 5 --- projects/composablekernel/groovy/vars/ck.groovy | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index edea323042a0..dbee8f89511b 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1514,16 +1514,11 @@ def runDispatcherPerfTests(String compiler) { -D BATCHED_GEMM_LAYOUT="rcr" \ -D BATCHED_CONTRACTION_DATATYPE="fp16" \ -D BATCHED_CONTRACTION_LAYOUT="rcr" \ - -D TILE_ENGINE_SAMPLING_TIER=daily \ - -D CK_TILE_DISPATCHER=ON \ - -D BUILD_DISPATCHER_BINDINGS=ON \ - -D DISPATCHER_RULE_SET=tests .. && \ - ninja -j${nthreads()} benchmark_gemm_universal_all benchmark_batched_gemm_all benchmark_batched_contraction_all ck_tile_dispatcher dispatcher_gemm_lib && \ + -D TILE_ENGINE_SAMPLING_TIER=daily .. && \ + ninja -j${nthreads()} benchmark_gemm_universal_all benchmark_batched_gemm_all benchmark_batched_contraction_all && \ python3 ../tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_universal_results.json && \ python3 ../tile_engine/ops/gemm/batched_gemm/batched_gemm_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json batched_gemm_results.json && \ - python3 ../tile_engine/ops/gemm/batched_contraction/batched_contraction_benchmark.py . --problem-configs "g=2;m=1024;n=1024;k=1024" --warmup 5 --repeat 5 --verbose --json batched_contraction_results.json && \ - python3 ../dispatcher/tests/test_batched_gemm_gpu_correctness.py --gfx gfx942 && \ - python3 ../dispatcher/tests/test_batched_contraction_gpu_correctness.py --gfx gfx942""" + python3 ../tile_engine/ops/gemm/batched_contraction/batched_contraction_benchmark.py . --problem-configs "g=2;m=1024;n=1024;k=1024" --warmup 5 --repeat 5 --verbose --json batched_contraction_results.json""" try { buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) } finally { From 85a0f7633b73f87654585a21b0d3c26957f4babb Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Mon, 24 Aug 2026 19:55:50 -0400 Subject: [PATCH 18/51] ci(ck-tile): split dispatcher CI into smoke and daily tiers Replace RUN_DISPATCHER_PERF_TESTS / RUN_DISPATCHER_CORRECTNESS_TESTS with four params, tiered by coverage and budget rather than by operator: RUN_DISPATCHER_PERF_TESTS gfx942, fp16/rcr, 1 size, budget 500 RUN_DISPATCHER_PERF_TESTS_DAILY gfx942, 4x4 matrix, 3 sizes, budget 8000 RUN_DISPATCHER_CORRECTNESS_TESTS gfx942, sweep budget 64 + fixed configs RUN_DISPATCHER_CORRECTNESS_TESTS_DAILY gfx942 + gfx950, budget 500 + variants The smoke tiers exist to catch codegen/build/launch breakage on demand without booking a GPU node for a full sweep. Correctness smoke keeps the full 4x4 dtype/layout lists on purpose: the sweep samples per stratum, so budget 64 spreads ~4 configs across all 16 combinations rather than testing one combination deeply. It stays gfx942-only because gfx950 is where the quant ops and the OCP-vs-FNUZ fp8 split live. runDispatcherPerfTests, runDispatcherCorrectnessTests and dispatcherVariantCmd take a tier argument. Result filenames carry it: both lanes write into projects/composablekernel/build, so without a suffix a build with both tiers enabled would have the second stage's archiveArtifacts overwrite the first's. Wire the two _DAILY params into the 22:00 develop cron so nightly dispatcher coverage is preserved across the split. Drop the gfx950 rowcolquant/tensorquant invocations: dispatcher/tests carries no test_rowcolquant_gpu_correctness.py or test_tensorquant_gpu_correctness.py, so those calls would fail the lane on "can't open file" before reaching a GPU. A comment records where to add them back. Co-Authored-By: Claude Opus 5 --- projects/composablekernel/Jenkinsfile | 83 +++++++++++-- .../composablekernel/groovy/vars/ck.groovy | 112 ++++++++++++------ 2 files changed, 152 insertions(+), 43 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index 66c46c1a6748..20191fa1ad86 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -37,7 +37,7 @@ def loadCk() { //launch develop branch daily jobs CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;RUN_CK_TILE_FMHA_TESTS=true;RUN_PERFORMANCE_TESTS=true;FORCE_CI=true - 0 22 * * * % RUN_FULL_QA=true;DISABLE_DL_KERNELS=true;RUN_TILE_ENGINE_BASIC_TESTS=true;RUN_TILE_ENGINE_GEMM_TESTS=true;RUN_DISPATCHER_PERF_TESTS=true;RUN_DISPATCHER_CORRECTNESS_TESTS=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true + 0 22 * * * % RUN_FULL_QA=true;DISABLE_DL_KERNELS=true;RUN_TILE_ENGINE_BASIC_TESTS=true;RUN_TILE_ENGINE_GEMM_TESTS=true;RUN_DISPATCHER_PERF_TESTS_DAILY=true;RUN_DISPATCHER_CORRECTNESS_TESTS_DAILY=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true 0 21 * * * % RUN_GROUPED_CONV_LARGE_CASES_TESTS=true;hipTensor_test=true;BUILD_GFX101=false;BUILD_GFX908=false;BUILD_GFX942=true;BUILD_GFX950=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true;BUILD_PACKAGES=true 0 19 * * * % BUILD_DOCKER=true;COMPILER_VERSION=develop;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true 0 17 * * * % BUILD_DOCKER=true;COMPILER_VERSION=therock;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true @@ -138,11 +138,19 @@ pipeline { booleanParam( name: "RUN_DISPATCHER_PERF_TESTS", defaultValue: false, - description: "Benchmark ahead-of-time tile_engine GEMM instances on gfx942: gemm_universal, batched_gemm, batched_contraction (default: OFF)") + description: "Smoke-tier benchmark of ahead-of-time tile_engine GEMM instances on gfx942: gemm_universal, batched_gemm, batched_contraction at fp16/rcr, one problem size, 500-instance budget (default: OFF)") + booleanParam( + name: "RUN_DISPATCHER_PERF_TESTS_DAILY", + defaultValue: false, + description: "Full-tier benchmark of ahead-of-time tile_engine GEMM instances on gfx942: the same three ops across the fp8/fp16/bf8/bf16 x rcr/rrr/crr/ccr matrix, three problem sizes, 8000-instance budget (default: OFF)") booleanParam( name: "RUN_DISPATCHER_CORRECTNESS_TESTS", defaultValue: false, - description: "Verify JIT-compiled dispatcher kernels on gfx942 + gfx950: gemm_universal, grouped, multi_d, multi_abd, stream_k, batched_gemm, batched_contraction, plus bquant + mx_gemm on gfx950 (per-arch subset; default: OFF)") + description: "Smoke-tier verification of JIT-compiled dispatcher kernels on gfx942: gemm_universal sweep at budget 64 plus parity, batched_gemm and batched_contraction (default: OFF)") + booleanParam( + name: "RUN_DISPATCHER_CORRECTNESS_TESTS_DAILY", + defaultValue: false, + description: "Full-tier verification of JIT-compiled dispatcher kernels on gfx942 + gfx950: gemm_universal sweep at budget 500 plus grouped, multi_d, multi_abd, batched_gemm, batched_contraction, stream_k on gfx942, and bquant + mx_gemm + rowcolquant + tensorquant on gfx950 (per-arch subset; default: OFF)") booleanParam( name: "BUILD_INSTANCES_ONLY", defaultValue: false, @@ -636,7 +644,35 @@ pipeline { loadCk() ck.runOnHealthyNode(rocmnode("gfx942")) { deleteDir() - ck.runDispatcherPerfTests(params.BUILD_COMPILER) + ck.runDispatcherPerfTests(params.BUILD_COMPILER, "smoke") + cleanWs() + } + } + } + } + } + } + stage("Run DISPATCHER_PERF_DAILY Tests") + { + when { + beforeAgent true + expression { env.SHOULD_RUN_CI.toBoolean() } + } + parallel + { + stage("Run DISPATCHER_PERF_DAILY Tests on gfx942") + { + when { + beforeAgent true + expression { params.RUN_DISPATCHER_PERF_TESTS_DAILY.toBoolean() } + } + agent none + steps{ + script { + loadCk() + ck.runOnHealthyNode(rocmnode("gfx942")) { + deleteDir() + ck.runDispatcherPerfTests(params.BUILD_COMPILER, "daily") cleanWs() } } @@ -708,6 +744,9 @@ pipeline { } } } + // Smoke tier is gfx942-only on purpose: gfx950 is where the quant ops and + // the OCP-vs-FNUZ fp8 split live, which is full-tier territory, and a + // smoke lane that books two GPU node types is no longer a smoke lane. stage("Run DISPATCHER_CORRECTNESS Tests") { when { @@ -728,17 +767,45 @@ pipeline { loadCk() ck.runOnHealthyNode(rocmnode("gfx942")) { deleteDir() - ck.runDispatcherCorrectnessTests("gfx942", params.BUILD_COMPILER) + ck.runDispatcherCorrectnessTests("gfx942", params.BUILD_COMPILER, "smoke") cleanWs() } } } } - stage("Run DISPATCHER_CORRECTNESS Tests on gfx950") + } + } + stage("Run DISPATCHER_CORRECTNESS_DAILY Tests") + { + when { + beforeAgent true + expression { env.SHOULD_RUN_CI.toBoolean() } + } + parallel + { + stage("Run DISPATCHER_CORRECTNESS_DAILY Tests on gfx942") { when { beforeAgent true - expression { params.RUN_DISPATCHER_CORRECTNESS_TESTS.toBoolean() } + expression { params.RUN_DISPATCHER_CORRECTNESS_TESTS_DAILY.toBoolean() } + } + agent none + steps{ + script { + loadCk() + ck.runOnHealthyNode(rocmnode("gfx942")) { + deleteDir() + ck.runDispatcherCorrectnessTests("gfx942", params.BUILD_COMPILER, "daily") + cleanWs() + } + } + } + } + stage("Run DISPATCHER_CORRECTNESS_DAILY Tests on gfx950") + { + when { + beforeAgent true + expression { params.RUN_DISPATCHER_CORRECTNESS_TESTS_DAILY.toBoolean() } } agent none steps{ @@ -746,7 +813,7 @@ pipeline { loadCk() ck.runOnHealthyNode(rocmnode("gfx950")) { deleteDir() - ck.runDispatcherCorrectnessTests("gfx950", params.BUILD_COMPILER) + ck.runDispatcherCorrectnessTests("gfx950", params.BUILD_COMPILER, "daily") cleanWs() } } diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index dbee8f89511b..617684634b4b 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1501,24 +1501,45 @@ def runTileEngineGemmTests(String arch, String compiler) { buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) } -def runDispatcherPerfTests(String compiler) { +// Benchmark ahead-of-time tile_engine instances. Two coverage tiers: +// +// "smoke" -- one dtype, one layout, one problem size, and a 500-instance AOT +// budget. Sized to catch codegen/build/launch breakage on demand +// without booking a GPU node for the length of a full sweep. +// "daily" -- the full 4x4 dtype x layout matrix across three problem sizes at +// TILE_ENGINE_SAMPLING_TIER=daily, which maps to an 8000-instance +// budget in tile_engine/ops/gemm/CMakeLists.txt. +// +// The tier is woven into every result filename. Both lanes write into +// projects/composablekernel/build, so without it a build with both enabled +// would have the second stage's archiveArtifacts overwrite the first's. +def runDispatcherPerfTests(String compiler, String tier) { + def daily = (tier == "daily") + def samplingTier = daily ? "daily" : "500" + def gemmDatatype = daily ? "fp8;fp16;bf8;bf16" : "fp16" + def gemmLayout = daily ? "rcr;rrr;crr;ccr" : "rcr" + // nargs="+" on both drivers, so extra sizes are extra positional args. + def problemSizes = daily ? '"1024,1024,1024" "2048,2048,2048" "4096,4096,4096"' + : '"1024,1024,1024"' + def problemConfigs = daily ? '"g=2;m=1024;n=1024;k=1024" "g=4;m=2048;n=2048;k=2048"' + : '"g=2;m=1024;n=1024;k=1024"' def execute_cmd = """ cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ -D BUILD_CK_TILE_ENGINE="ON" \ -D CMAKE_CXX_COMPILER="${compiler}" \ -D CMAKE_BUILD_TYPE=Release \ -D GPU_TARGETS="gfx942" \ - -D GEMM_UNIVERSAL_DATATYPE="fp8;fp16;bf8;bf16" \ - -D GEMM_UNIVERSAL_LAYOUT="rcr;rrr;crr;ccr" \ + -D GEMM_UNIVERSAL_DATATYPE="${gemmDatatype}" \ + -D GEMM_UNIVERSAL_LAYOUT="${gemmLayout}" \ -D BATCHED_GEMM_DATATYPE="fp16" \ -D BATCHED_GEMM_LAYOUT="rcr" \ -D BATCHED_CONTRACTION_DATATYPE="fp16" \ -D BATCHED_CONTRACTION_LAYOUT="rcr" \ - -D TILE_ENGINE_SAMPLING_TIER=daily .. && \ + -D TILE_ENGINE_SAMPLING_TIER=${samplingTier} .. && \ ninja -j${nthreads()} benchmark_gemm_universal_all benchmark_batched_gemm_all benchmark_batched_contraction_all && \ - python3 ../tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_universal_results.json && \ - python3 ../tile_engine/ops/gemm/batched_gemm/batched_gemm_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json batched_gemm_results.json && \ - python3 ../tile_engine/ops/gemm/batched_contraction/batched_contraction_benchmark.py . --problem-configs "g=2;m=1024;n=1024;k=1024" --warmup 5 --repeat 5 --verbose --json batched_contraction_results.json""" + python3 ../tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_universal_results_${tier}.json && \ + python3 ../tile_engine/ops/gemm/batched_gemm/batched_gemm_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json batched_gemm_results_${tier}.json && \ + python3 ../tile_engine/ops/gemm/batched_contraction/batched_contraction_benchmark.py . --problem-configs ${problemConfigs} --warmup 5 --repeat 5 --verbose --json batched_contraction_results_${tier}.json""" try { buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) } finally { @@ -1527,9 +1548,7 @@ def runDispatcherPerfTests(String compiler) { // reasoning as runDispatcherCorrectnessTests: execute_cmd runs from // projects/composablekernel/build while archiveArtifacts resolves against // the workspace root, so the build-relative path is spelled out in full. - archiveArtifacts artifacts: "projects/composablekernel/build/gemm_universal_results.json," + - "projects/composablekernel/build/batched_gemm_results.json," + - "projects/composablekernel/build/batched_contraction_results.json", + archiveArtifacts artifacts: "projects/composablekernel/build/*_results_${tier}.json", allowEmptyArchive: true } } @@ -1564,7 +1583,7 @@ def dispatcherGemmVariantsFor(String arch) { // runner's default --elementwise-op PassThrough. The budget is well under the // standard lane's 500 because each variant JIT-compiles its own .so set and // these run in addition to, not instead of, the standard sweep. -def dispatcherVariantCmd(String arch, String variant) { +def dispatcherVariantCmd(String arch, String variant, String tier) { return """python3 ../dispatcher/tests/test_gemm_search_space.py \ --variant ${variant} \ --arch ${arch} \ @@ -1572,10 +1591,24 @@ def dispatcherVariantCmd(String arch, String variant) { --warmup 5 \ --repeat 5 \ --size 1024 \ - --json dispatcher_${variant}_results.json""" + --json dispatcher_${variant}_results_${tier}.json""" } -def runDispatcherCorrectnessTests(String arch, String compiler) { +// Verify JIT-compiled dispatcher kernels against a host reference. Two tiers: +// +// "smoke" -- the standard sweep at --budget 64 plus the three fixed-config +// tests (parity, batched_gemm, batched_contraction). The dtype and +// layout lists stay at the full 4x4 on purpose: the sweep samples +// per stratum, so 64 spreads ~4 configs across all 16 dtype x +// layout combinations rather than testing one combination deeply. +// "daily" -- the same, at --budget 500, plus every non-standard GEMM variant +// this arch can run and the arch-specific extras below. +// +// As in runDispatcherPerfTests, result filenames carry the tier so a build with +// both lanes enabled does not have one stage's artifacts clobber the other's. +def runDispatcherCorrectnessTests(String arch, String compiler, String tier) { + def daily = (tier == "daily") + def budget = daily ? 500 : 64 def execute_cmd = """ cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ -D CMAKE_CXX_COMPILER="${compiler}" \ @@ -1589,35 +1622,44 @@ def runDispatcherCorrectnessTests(String arch, String compiler) { --arch ${arch} \ --dtypes fp16,bf16,fp8,bf8 \ --layouts rcr,rrr,crr,ccr \ - --budget 500 \ + --budget ${budget} \ --warmup 5 \ --repeat 5 \ --size 1024 \ - --json dispatcher_gemm_results.json && \ + --json dispatcher_gemm_results_${tier}.json && \ python3 ../dispatcher/tests/test_gemm_parity.py && \ python3 ../dispatcher/tests/test_batched_gemm_gpu_correctness.py --gfx ${arch} && \ python3 ../dispatcher/tests/test_batched_contraction_gpu_correctness.py --gfx ${arch}""" - dispatcherGemmVariantsFor(arch).each { variant -> - execute_cmd += " && \\\n " + dispatcherVariantCmd(arch, variant) - } - // Stream-K has its own registry-level driver test; its defaults are the - // full 4x4 matrix (48 driver compiles / 96 GPU runs), so CI pins one - // dtype/layout. Note it takes --arch, not --gfx. - if (arch == "gfx942") { - execute_cmd += """ && \ + // Everything below is daily-only: each variant JIT-compiles its own .so set, + // which is the bulk of this lane's wall clock and the opposite of a smoke run. + if (daily) { + dispatcherGemmVariantsFor(arch).each { variant -> + execute_cmd += " && \\\n " + dispatcherVariantCmd(arch, variant, tier) + } + // Stream-K has its own registry-level driver test; its defaults are the + // full 4x4 matrix (48 driver compiles / 96 GPU runs), so CI pins one + // dtype/layout. Note it takes --arch, not --gfx. + if (arch == "gfx942") { + execute_cmd += """ && \ python3 ../dispatcher/tests/test_streamk_registry.py --arch ${arch} --datatypes fp16 --layouts rcr""" - } - // mx_gemm has no expand_sweep -- mx_gemm_utils exposes only - // default_fp8_config()/default_fp4_config() -- so it cannot join the - // --variant sweep above and instead runs the two-config smoke test merged - // in #10132. That test is unittest-based and self-gates to gfx950, so it - // takes no --gfx (unlike test_bquant, which does). - if (arch == "gfx950") { - execute_cmd += """ && \ + } + // mx_gemm has no expand_sweep -- mx_gemm_utils exposes only + // default_fp8_config()/default_fp4_config() -- so it cannot join the + // --variant sweep above and instead runs the two-config smoke test merged + // in #10132. That test is unittest-based and self-gates to gfx950, so it + // takes no --gfx (unlike test_bquant, which does). + // + // rowcolquant and tensorquant are deliberately absent: the tile_engine + // side of both ops exists, but dispatcher/tests carries no + // test_rowcolquant_gpu_correctness.py / test_tensorquant_gpu_correctness.py + // yet, so invoking them here would fail the lane on "can't open file" + // before reaching a GPU. Add the calls back in the same change that adds + // the tests. + if (arch == "gfx950") { + execute_cmd += """ && \ python3 ../dispatcher/tests/test_bquant_gpu_correctness.py --gfx ${arch} && \ - python3 ../dispatcher/tests/test_mx_gemm_gpu_correctness.py && \ - python3 ../dispatcher/tests/test_rowcolquant_gpu_correctness.py --gfx ${arch} && \ - python3 ../dispatcher/tests/test_tensorquant_gpu_correctness.py --gfx ${arch}""" + python3 ../dispatcher/tests/test_mx_gemm_gpu_correctness.py""" + } } try { buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) @@ -1630,7 +1672,7 @@ def runDispatcherCorrectnessTests(String arch, String compiler) { // against the workspace root. A bare "dispatcher_*_results.json" matches // nothing there, and allowEmptyArchive hides that it matched nothing. // Glob, not one name: each variant writes its own dispatcher__*.json. - archiveArtifacts artifacts: "projects/composablekernel/build/dispatcher_*_results.json", + archiveArtifacts artifacts: "projects/composablekernel/build/dispatcher_*_results_${tier}.json", allowEmptyArchive: true } } From f99bb708a03baf7699cfb440c0b58a39a66648a1 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Tue, 25 Aug 2026 15:34:14 -0400 Subject: [PATCH 19/51] fix(ck-tile): match aquant hipcc defines to bquant, correct stale streamk note _compile_aquant_kernel omitted the arch-specific defines its bquant sibling passes (grouped_gemm_bquant_utils.py:524-531), so a standalone aquant build on gfx950/gfx12 compiled without CK_USE_OCP_FP8 / CK_TILE_USE_OCP_FP8 and without CK_USE_NATIVE_MX_SUPPORT / CK_GFX950_SUPPORT. CMake injects these on the normal build path; the hipcc path in this module did not. This is latent, not an active breakage. For the default fp8/bf8 configs the AQ scale is float32 and A/B cross the ctypes boundary as raw bytes, so nothing converts fp8 on the host today. The exposure is the fp8i4/bf8i4 variants, where AQ is itself fp8 and would be encoded FNUZ by a host that believes it is not on an OCP part. Making the define explicit removes the footgun and keeps the two quant modules identical on the build path. Separately, streamk_gemm_ctypes_lib.cpp claimed its generated launch "uses the Atomic reduction strategy". It does not: the strategy is a codegen-time template constant baked into SelectedKernel (unified_gemm_codegen.py:1225), and GetWorkSpaceSize returns 0 for Atomic and non-zero for Linear/Tree, so the one allocate-and-launch path serves all three. The lib is strategy agnostic; the comment now says so. Co-Authored-By: Claude Opus 5 --- .../bindings/ctypes/streamk_gemm_ctypes_lib.cpp | 7 +++++-- .../dispatcher/python/grouped_gemm_aquant_utils.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/streamk_gemm_ctypes_lib.cpp b/projects/composablekernel/dispatcher/bindings/ctypes/streamk_gemm_ctypes_lib.cpp index 11b7fc44f005..37ed77230ef6 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/streamk_gemm_ctypes_lib.cpp +++ b/projects/composablekernel/dispatcher/bindings/ctypes/streamk_gemm_ctypes_lib.cpp @@ -14,8 +14,11 @@ * * static float launch(const ck_tile::StreamKHostArgs& args, const stream_config& stream); * - * which allocates the reduction workspace internally (DeviceMem) and uses the - * Atomic reduction strategy. The single-problem registry path + * which allocates the reduction workspace internally (DeviceMem). This lib is + * reduction-strategy agnostic: the strategy is a codegen-time template constant + * baked into SelectedKernel (unified_gemm_codegen.py), and GetWorkSpaceSize + * returns 0 for Atomic and a non-zero size for Linear/Tree, so the single + * allocate-and-launch path below serves all three. The single-problem registry path * (g_dispatcher->run / GemmHostArgs) and the generated_tile_backend wrapper both * hard-code the plain GemmHostArgs launch, so this lib bypasses the registry and * calls SelectedKernel::launch(args, stream) directly, reporting the kernel name diff --git a/projects/composablekernel/dispatcher/python/grouped_gemm_aquant_utils.py b/projects/composablekernel/dispatcher/python/grouped_gemm_aquant_utils.py index 60e529446e2c..f36524c780ea 100644 --- a/projects/composablekernel/dispatcher/python/grouped_gemm_aquant_utils.py +++ b/projects/composablekernel/dispatcher/python/grouped_gemm_aquant_utils.py @@ -498,9 +498,20 @@ def _compile_aquant_kernel( """Compile a generated .hpp into a .so via hipcc. Returns True on success.""" ck_include = _get_ck_include_dir() + # Arch-specific defines: gfx950 uses OCP fp8 (not FNUZ) and native MX support. + # These mirror the CMakeLists.txt definitions that are normally injected by CMake + # but are absent in the standalone hipcc build path. Same block as + # grouped_gemm_bquant_utils._compile_bquant_kernel. + arch_defines = [] + if "gfx12" in gfx_arch or "gfx950" in gfx_arch: + arch_defines += ["-DCK_USE_OCP_FP8", "-DCK_TILE_USE_OCP_FP8"] + if "gfx950" in gfx_arch: + arch_defines += ["-DCK_USE_NATIVE_MX_SUPPORT", "-DCK_GFX950_SUPPORT"] + cmd = [hipcc] + _HIPCC_BASE_FLAGS + [ f"--offload-arch={gfx_arch}", f"-DGFX_ARCH=\"{gfx_arch}\"", + *arch_defines, "-include", str(hpp_path), str(_CTYPES_LIB_SRC), "-o", str(so_path), From 646380875d2fbf8816f033550932dccfa6085a50 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Tue, 25 Aug 2026 15:38:16 -0400 Subject: [PATCH 20/51] test(ck-tile): add GPU correctness tests for stream-K and aquant GEMM Two dispatcher operators had no on-device numeric verification through the ctypes bridge. grouped_gemm_aquant had none at all (CPU unit tests only). gemm_streamk was covered by test_gemm_search_space.py --variant stream_k, but for the atomic reduction strategy only: default_ci_config.json carries no streamk_config block, so expand_sweep hits the ["atomic"] fallback (gemm_utils.py:2245-2249) and linear/tree were never launched from Python. test_streamk_gpu_correctness.py builds 12 kernels (fp16/bf16/fp8/bf8 x atomic/linear/tree) and runs 20 cases across two shapes: S1 4096x4096x2048 all three strategies; output tiles vastly exceed occupancy, so atomic's num_wgs_per_tile stays <= 2 S2 128x128x8192 linear/tree ONLY. estimate_num_wgs_per_tile is gated on Atomic (streamk_gemm_tile_partitioner_impl.hpp:279) and returns 1 otherwise, so linear/tree keep a tight bound in the SK-heavy regime where atomic would need ~5e-1 rtol for bf16 -- a vacuous assertion. The atomic side of that regime is already covered in C++ by test_streamk_registry.py. num_wgs_per_tile is not knowable from Python (it needs hipOccupancyMaxActiveBlocksPerMultiprocessor), which is why the shapes are chosen to pin it to a provable bound instead of guessing. Tolerance is a port of CK's own split-K-aware formula from examples/gemm/cpp/streamk_driver_common.hpp:85-101 plus get_relative_threshold / get_absolute_threshold in host/check_err.hpp -- there was no Python equivalent. Asserted element-wise, not as a scalar max-relative-error; the scalar is kept for logging only. atol scales with the reference's max magnitude, so it is computed at runtime. test_aquant_gpu_correctness.py mirrors test_bquant_gpu_correctness.py: fp8, bf8 and a 64x256 tiled case, with the reference applying the A-side per-(m,k)-group scale loop before the fp32 matmul. K=3*TileK exercises TailNumber::Odd. It skips (77) if ml_dtypes is missing rather than falling back to a stand-in codec -- comparing against knowingly wrong values is worse than not comparing. Both are script-style (argparse, exit 77 to skip) and self-gate by arch: stream-K to gfx942 (gfx90a has no fp8/bf8 MFMA; on gfx950 the FNUZ host codecs in gemm_utils disagree with the arch's OCP encoding), aquant to gfx950. Both set CK_TILE_BENCH_WARMUP/REPEAT low -- the default 50/100 is pure waste for a correctness run. Not yet exercised on hardware: authored and validated on a CPU-only box. The tolerance port was checked against hand-computed values and the skip paths verified; the GPU runs themselves are unconfirmed. Co-Authored-By: Claude Opus 5 --- .../tests/test_aquant_gpu_correctness.py | 309 ++++++++++++++ .../tests/test_streamk_gpu_correctness.py | 396 ++++++++++++++++++ 2 files changed, 705 insertions(+) create mode 100644 projects/composablekernel/dispatcher/tests/test_aquant_gpu_correctness.py create mode 100644 projects/composablekernel/dispatcher/tests/test_streamk_gpu_correctness.py diff --git a/projects/composablekernel/dispatcher/tests/test_aquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_aquant_gpu_correctness.py new file mode 100644 index 000000000000..dea681b55bc8 --- /dev/null +++ b/projects/composablekernel/dispatcher/tests/test_aquant_gpu_correctness.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 + +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +""" +GPU correctness tests for AQuant GEMM dispatcher — A-side per-group scales. + +Companion to test_bquant_gpu_correctness.py. Where BQuant scales the B operand +per (k, n) group, AQuant scales A per (m, k) group: + + C = dequant(A, AQ) @ B, A[m, k] *= AQ[m // gM, k // gK] + +Requires a gfx950 GPU (MI350X / MI355X) and hipcc in PATH. Skips cleanly +(exit 77) when no GPU is visible or the detected arch is not gfx950, so it is +safe to invoke unconditionally from a CI lane. + +Tests: + fp8, bf8 — GPU output is non-zero, finite, and within 5% max-relative-error + vs. a fp32 CPU reference; time_ms is positive. + fp8/tiled — same checks on a shape spanning a 4x4 grid of output tiles + (tile_m=16, tile_n=64), which the 1x1-tile cases cannot catch. + +Run: + python3 test_aquant_gpu_correctness.py + python3 test_aquant_gpu_correctness.py -v # verbose hipcc output + python3 test_aquant_gpu_correctness.py --gfx gfx950 +""" + +import argparse +import logging +import math +import sys +import tempfile +from pathlib import Path + +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "python")) + +from grouped_gemm_aquant_utils import ( + AQuantGemmProblem, + AQuantGpuGemmRunner, + setup_multiple_aquant_dispatchers, + default_fp8_config, + default_bf8_config, +) + +log = logging.getLogger(__name__) + +TOLERANCE = 0.05 # 5% max relative error — fp8/bf8 precision floor + +PASS = "PASS" +FAIL = "FAIL" + +# ctest reports this as "skipped" rather than passed. Without it, a CPU-only or +# non-gfx950 runner would report every AQuant test as a hard FAIL for a reason +# that has nothing to do with the code under test. +SKIP_EXIT = 77 + +# AQuant shares BQuant's gfx950 (MI350X/MI355X) quantised-MFMA requirement. +SUPPORTED_ARCHS = ("gfx950",) + + +# --------------------------------------------------------------------------- +# Dtype helpers +# --------------------------------------------------------------------------- + +def _encode_fp8(arr: np.ndarray, dtype: str) -> np.ndarray: + """Encode float32 → fp8 bytes (uint8 view of the fp8 bit pattern). + + fp8 = OCP e4m3fn, bf8 = OCP e5m2. Kernels on gfx950 are compiled with + -DCK_TILE_USE_OCP_FP8 (see grouped_gemm_aquant_utils._compile_aquant_kernel), + so the bit patterns match what the kernel decodes. + """ + import ml_dtypes + + ml_t = ml_dtypes.float8_e4m3fn if dtype == "fp8" else ml_dtypes.float8_e5m2 + return arr.astype(ml_t).view(np.uint8) + + +def _decode_fp8(arr: np.ndarray, dtype: str) -> np.ndarray: + """Decode fp8 bytes (uint8 view) back to float32.""" + import ml_dtypes + + ml_t = ml_dtypes.float8_e4m3fn if dtype == "fp8" else ml_dtypes.float8_e5m2 + return arr.view(ml_t).astype(np.float32) + + +# --------------------------------------------------------------------------- +# CPU reference +# --------------------------------------------------------------------------- + +def _reference_gemm(A_f32: np.ndarray, B_f32: np.ndarray, + AQ: np.ndarray, problem: AQuantGemmProblem) -> np.ndarray: + """C = dequant(A, AQ) @ B in fp32. + + A is scaled per (m, k) group — the transpose of BQuant's (k, n) grouping. + Mirrors reference_aquant_gemm in examples/gemm/python/14_grouped_gemm_aquant.py. + """ + gM, gK = problem.quant_group_m, problem.quant_group_k + A_dq = A_f32.astype(np.float32).copy() + for qi in range(problem.QM_A): + for qk in range(problem.QK_A): + m0, m1 = qi * gM, min((qi + 1) * gM, problem.M) + k0, k1 = qk * gK, min((qk + 1) * gK, problem.K) + A_dq[m0:m1, k0:k1] *= float(AQ[qi, qk]) + return (A_dq @ B_f32.astype(np.float32)).astype(np.float32) + + +def _max_rel_err(C_gpu: np.ndarray, C_ref: np.ndarray) -> float: + C_gpu_f = C_gpu.astype(np.float32) + C_ref_f = C_ref.astype(np.float32) + num = np.abs(C_gpu_f - C_ref_f) + # Use 1% of the global max magnitude as the denominator floor to avoid + # inflating the relative error when individual elements are near zero + # (a common occurrence with random inputs that partially cancel in GEMM). + ref_max = float(np.abs(C_ref_f).max()) + den = np.abs(C_ref_f) + max(ref_max * 1e-2, 1e-6) + return float(np.max(num / den)) + + +# --------------------------------------------------------------------------- +# Core test helper +# --------------------------------------------------------------------------- + +def _detect_arch() -> "str | None": + """Detected gfx arch, or None when no GPU is visible. + + The empty fallback keeps absence detectable; detect_gpu_arch would otherwise + invent a default and turn "no GPU" into a confusing build failure. + """ + try: + from dispatcher_common import detect_gpu_arch + + return detect_gpu_arch(fallback="") or None + except Exception: + return None + + +def _make_inputs(M, N, K, gM, gK, dtype, seed=42): + """Generate fp8-encoded A/B plus float32 AQ scales, and the decoded references. + + A and B are encoded to fp8 and decoded back so the CPU reference consumes the + same rounded values the kernel does; otherwise the comparison would be + dominated by quantisation error rather than kernel correctness. + """ + rng = np.random.default_rng(seed) + QM_A = math.ceil(M / gM) + QK_A = math.ceil(K / gK) + A_f32 = rng.uniform(-2.0, 2.0, (M, K)).astype(np.float32) + B_f32 = rng.uniform(-2.0, 2.0, (K, N)).astype(np.float32) + AQ = rng.uniform(0.5, 2.0, (QM_A, QK_A)).astype(np.float32) + A_raw = _encode_fp8(A_f32, dtype) + B_raw = _encode_fp8(B_f32, dtype) + return A_raw, _decode_fp8(A_raw, dtype), B_raw, _decode_fp8(B_raw, dtype), AQ + + +def _run_one(label: str, config, M: int, N: int, K: int, + A_raw: np.ndarray, A_f32: np.ndarray, + B_raw: np.ndarray, B_f32: np.ndarray, + AQ: np.ndarray, + out_dir: Path, + gfx_arch: str = "gfx950") -> "tuple[str, str]": + """Build, run, and verify one kernel. Returns (PASS|FAIL, detail_message).""" + problem = AQuantGemmProblem( + M=M, N=N, K=K, + quant_group_m=config.quant_group_m, + quant_group_n=config.quant_group_n, + quant_group_k=config.quant_group_k, + ) + + so_paths = setup_multiple_aquant_dispatchers( + configs=[config], + output_dir=out_dir, + gfx_arch=gfx_arch, + ) + if not so_paths or so_paths[0] is None: + return FAIL, f"{label}: kernel build failed" + + runner = AQuantGpuGemmRunner(so_paths[0]) + + result = runner.run(A=A_raw, B=B_raw, AQ=AQ, problem=problem, c_dtype=np.float16) + C_gpu = result.C.astype(np.float32) + + # An all-zero C is the signature of a kernel that launched but never wrote — + # e.g. an MFMA instruction unavailable on the target arch. + if np.all(C_gpu == 0): + return FAIL, f"{label}: GPU output is all-zero" + nan_mask = ~np.isfinite(C_gpu) + if np.any(nan_mask): + nan_frac = nan_mask.mean() + log.debug("%s: NaN/Inf fraction=%.3f, first 8 elements=%s", + label, nan_frac, C_gpu.flat[:8].tolist()) + return FAIL, f"{label}: GPU output contains NaN/Inf (frac={nan_frac:.3f})" + + C_ref = _reference_gemm(A_f32, B_f32, AQ, problem) + mre = _max_rel_err(C_gpu, C_ref) + if mre > TOLERANCE: + return FAIL, (f"{label}: max_rel_err={mre:.4f} > tol={TOLERANCE:.4f} " + f"(shape M={M} N={N} K={K})") + + # Timing sanity: a zero time_ms means the HIP event pair never bracketed a launch. + result_timed = runner.run(A=A_raw, B=B_raw, AQ=AQ, problem=problem, + c_dtype=np.float16) + if result_timed.time_ms <= 0.0: + return FAIL, f"{label}: time_ms={result_timed.time_ms:.4f} is not positive" + + return PASS, (f"{label}: max_rel_err={mre:.4f}, " + f"time_ms={result_timed.time_ms:.3f}") + + +# --------------------------------------------------------------------------- +# Individual test cases +# --------------------------------------------------------------------------- + +def test_fp8(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": + # K=768 = 3*TileK(256): num_loop=3 gives the no-hot-loop/TailNumber::Odd path, + # which exercises the AQ scale prefetch more thoroughly than an even tail. + M, N, K, gM, gK = 16, 64, 768, 1, 128 + cfg = default_fp8_config(quant_group_k=gK, quant_group_m=gM, gfx_arch=gfx_arch) + A_raw, A_dec, B_raw, B_dec, AQ = _make_inputs(M, N, K, gM, gK, "fp8") + return _run_one("fp8", cfg, M, N, K, A_raw, A_dec, B_raw, B_dec, AQ, + out_dir, gfx_arch=gfx_arch) + + +def test_bf8(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": + M, N, K, gM, gK = 16, 64, 768, 1, 128 + cfg = default_bf8_config(quant_group_k=gK, quant_group_m=gM, gfx_arch=gfx_arch) + A_raw, A_dec, B_raw, B_dec, AQ = _make_inputs(M, N, K, gM, gK, "bf8", seed=43) + return _run_one("bf8", cfg, M, N, K, A_raw, A_dec, B_raw, B_dec, AQ, + out_dir, gfx_arch=gfx_arch) + + +def test_fp8_tiled(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": + # M=64, N=256 spans 4x4 output tiles (tile_m=16, tile_n=64). The single-tile + # cases above cannot distinguish a correct tile index from a constant one. + M, N, K, gM, gK = 64, 256, 768, 1, 128 + cfg = default_fp8_config(quant_group_k=gK, quant_group_m=gM, gfx_arch=gfx_arch) + A_raw, A_dec, B_raw, B_dec, AQ = _make_inputs(M, N, K, gM, gK, "fp8", seed=44) + return _run_one("fp8/tiled", cfg, M, N, K, A_raw, A_dec, B_raw, B_dec, AQ, + out_dir, gfx_arch=gfx_arch) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +TESTS = [ + ("fp8", test_fp8), + ("bf8", test_bf8), + ("fp8/tiled", test_fp8_tiled), +] + + +def main(): + parser = argparse.ArgumentParser(description="AQuant GPU correctness tests") + # No hardcoded default: it must stay possible to tell "user asked for + # gfx950" from "we are on an unrelated box", so the skip below can fire. + parser.add_argument("--gfx", default=None, + help="GPU arch override (default: auto-detect; gfx950 only)") + parser.add_argument("-v", "--verbose", action="store_true") + parser.add_argument("--output-dir", type=Path, default=None) + args = parser.parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(levelname)s: %(message)s", + ) + + gfx = args.gfx or _detect_arch() + if not gfx: + print("SKIP: no supported GPU detected (rocminfo); AQuant GPU tests skipped") + return SKIP_EXIT + if gfx not in SUPPORTED_ARCHS: + print(f"SKIP: AQuant is {'/'.join(SUPPORTED_ARCHS)}-only; detected {gfx}") + return SKIP_EXIT + try: + import ml_dtypes # noqa: F401 + except ImportError: + # Without ml_dtypes there is no trustworthy fp8 codec here, and a + # stand-in would compare the kernel against the wrong values. + print("SKIP: ml_dtypes not installed; AQuant fp8/bf8 encoding unavailable") + return SKIP_EXIT + + out_dir = args.output_dir or Path(tempfile.mkdtemp(prefix="aquant_gpu_test_")) + log.info("Kernel output dir: %s", out_dir) + + results = [] + for name, fn in TESTS: + log.info("--- Running %s ---", name) + try: + status, detail = fn(out_dir, gfx) + except Exception as exc: + status, detail = FAIL, f"{name}: exception: {exc}" + results.append((name, status, detail)) + log.info("[%s] %s", status, detail) + + print("\n=== Summary ===") + passed = sum(1 for _, s, _ in results if s == PASS) + for name, status, detail in results: + print(f" [{status:4s}] {detail}") + print(f"\n{passed}/{len(results)} passed") + + return 0 if passed == len(results) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/projects/composablekernel/dispatcher/tests/test_streamk_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_streamk_gpu_correctness.py new file mode 100644 index 000000000000..0790c432c827 --- /dev/null +++ b/projects/composablekernel/dispatcher/tests/test_streamk_gpu_correctness.py @@ -0,0 +1,396 @@ +#!/usr/bin/env python3 + +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +""" +GPU correctness tests for the Stream-K GEMM dispatcher — all three reduction +strategies. + +The shared sweep (``test_gemm_search_space.py --variant stream_k``) only ever +builds the ``atomic`` strategy: ``default_ci_config.json`` carries no +``streamk_config`` block, so ``expand_sweep`` falls back to ``["atomic"]``. +``linear`` and ``tree`` — which reduce through a device workspace instead of +global atomics — have never been verified through the ctypes bridge. This test +closes that gap. + +Requires a gfx942 GPU (MI300X) and hipcc in PATH. Skips cleanly (exit 77) when +no GPU is visible or the detected arch is not gfx942, so it is safe to invoke +unconditionally from a CI lane. + +Shapes: + S1 M=4096 N=4096 K=2048 — all three strategies. Output tiles vastly exceed + the occupancy limit, so the partitioner leaves few (if any) Stream-K + tiles and ``num_wgs_per_tile`` stays <= 2 under atomic. + S2 M=128 N=128 K=8192 — linear/tree ONLY. This is the Stream-K-heavy + regime: one output tile split across many workgroups. + ``estimate_num_wgs_per_tile`` is gated on the Atomic strategy + (streamk_gemm_tile_partitioner_impl.hpp:279) and returns 1 otherwise, so + linear/tree keep a tight tolerance here where atomic would need ~5e-1 + rtol for bf16 — a vacuous assertion. The atomic side of this regime is + already covered in C++ by test_streamk_registry.py. + +Tolerance is CK's own split-K-aware bound, ported from +``dispatcher/examples/gemm/cpp/streamk_driver_common.hpp`` + +``include/ck_tile/host/check_err.hpp`` — asserted element-wise, not as a +scalar max-relative-error. + +Run: + python3 test_streamk_gpu_correctness.py + python3 test_streamk_gpu_correctness.py -v # verbose hipcc + python3 test_streamk_gpu_correctness.py --gfx gfx942 + python3 test_streamk_gpu_correctness.py --dtypes fp16,bf16 # trim matrix +""" + +import argparse +import logging +import math +import os +import sys +from pathlib import Path + +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "python")) + +from ctypes_utils import detect_gpu_arch +from gemm_utils import ( + GemmKernelConfig, + GemmProblem, + GpuGemmRunner, + numpy_dtype_for, + output_dtype_for, + setup_multiple_gemm_dispatchers, +) + +log = logging.getLogger("streamk_gpu") + +PASS, FAIL = "PASS", "FAIL" +SKIP_EXIT = 77 + +# gfx942 only. gfx90a has no fp8/bf8 MFMA, and on gfx950 the fp8/bf8 host codecs +# in gemm_utils (FNUZ) disagree with the arch's OCP encoding — either arch would +# make half the matrix red for reasons unrelated to Stream-K. +SUPPORTED_ARCHS = ("gfx942",) + +STRATEGIES = ("atomic", "linear", "tree") +ALL_DTYPES = ("fp16", "bf16", "fp8", "bf8") + +# pad_m/n/k are all False, so every dimension must be a multiple of the tile. +TILE_M = TILE_N = TILE_K = 64 + +S1 = (4096, 4096, 2048) # dp-dominant: all strategies +S2 = (128, 128, 8192) # sk-dominant: linear/tree only + +# Upper bound on the workgroups reducing into a single output tile. Not knowable +# from Python (it needs hipOccupancyMaxActiveBlocksPerMultiprocessor), which is +# exactly why the shapes above pin it to a provable bound. +S1_ATOMIC_WGS_PER_TILE = 2 +NON_ATOMIC_WGS_PER_TILE = 1 + + +# --------------------------------------------------------------------------- +# Tolerance: a Python port of CK's split-K-aware bound +# --------------------------------------------------------------------------- + +# Explicit-mantissa bit counts, matching ck_tile::numeric::mant(): +# fp32 numeric.hpp:92 | fp16 half.hpp:233 | bf16 bfloat16.hpp:453 +# fp8/e4m3 float8.hpp:220 | bf8/e5m2 float8.hpp:238 +_MANT_BITS = {"fp32": 23, "fp16": 10, "bf16": 7, "fp8": 3, "bf8": 2} + + +def _relative_threshold(compute: str, out: str, acc: str, n_accum: int) -> float: + """Port of ck_tile::get_relative_threshold(n_accum).""" + compute_error = 2.0 ** -_MANT_BITS[compute] * 0.5 + output_error = 2.0 ** -_MANT_BITS[out] * 0.5 + midway_error = max(compute_error, output_error) + acc_error = 2.0 ** -_MANT_BITS[acc] * 0.5 * n_accum + return max(acc_error, midway_error) + + +def _absolute_threshold( + compute: str, out: str, acc: str, maxv: float, n_accum: int +) -> float: + """Port of ck_tile::get_absolute_threshold(maxv, n_accum).""" + if maxv == 0.0: + return 0.0 + expo = math.floor(math.log2(abs(maxv))) + compute_error = 2.0 ** (expo - _MANT_BITS[compute]) * 0.5 + # Full ULP, not half: the output error also absorbs the hardware-vs-software + # conversion difference. This asymmetry is deliberate in check_err.hpp. + output_error = 2.0 ** (expo - _MANT_BITS[out]) * 1.0 + midway_error = max(compute_error, output_error) + acc_error = 2.0 ** (expo - _MANT_BITS[acc]) * 0.5 * n_accum + return max(acc_error, midway_error) + + +def _streamk_tolerance(dtype: str, out_dtype: str, K: int, num_wgs_per_tile: int, + maxv: float): + """Port of streamk_tolerance() from streamk_driver_common.hpp:85-101. + + Stream-K accumulates a tile's K-range across ``num_wgs_per_tile`` partials, + so the bound is the looser of two regimes: the per-split MAC error (over + ``K / num_wgs`` accumulations at accumulator precision) and the cross-split + reduction error (over ``num_wgs`` accumulations at OUTPUT precision, which is + what atomics and the workspace both use). + """ + k_per_split = -(-K // num_wgs_per_tile) # integer ceil + rtol_base = _relative_threshold(dtype, out_dtype, "fp32", k_per_split) + atol_base = _absolute_threshold( + dtype, out_dtype, "fp32", maxv / num_wgs_per_tile, k_per_split + ) + rtol_split_k = _relative_threshold(out_dtype, out_dtype, out_dtype, + num_wgs_per_tile) + atol_split_k = _absolute_threshold(out_dtype, out_dtype, out_dtype, maxv, + num_wgs_per_tile) + return max(rtol_base, rtol_split_k), max(atol_base, atol_split_k) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _quantize(x: np.ndarray, dtype: str) -> np.ndarray: + """Round through ``dtype`` and back to fp32. + + The kernel consumes rounded operands, so the reference must too — otherwise + the comparison measures the quantization step, not the kernel. + """ + return x.astype(numpy_dtype_for(dtype)).astype(np.float32) + + +def _max_rel(got: np.ndarray, ref: np.ndarray) -> float: + """Global relative error, for logging only. The assertion is element-wise.""" + denom = float(np.max(np.abs(ref))) + if denom == 0.0: + return float(np.max(np.abs(got))) + return float(np.max(np.abs(got.astype(np.float32) - ref))) / denom + + +def _detect_arch(): + try: + arch = detect_gpu_arch(fallback="") + except Exception: + return None + return arch or None + + +def _make_config(dtype: str, strategy: str, gfx_arch: str) -> GemmKernelConfig: + """One Stream-K kernel config, spelled out rather than defaulted. + + Mirrors what the CI sweep builds green today for variant=stream_k on gfx942. + ``epilogue`` must stay "cshuffle": the codegen emits nothing for any other + value, which surfaces later as a confusing "no .hpp generated" build error. + """ + return GemmKernelConfig( + dtype_a=dtype, + dtype_b=dtype, + dtype_c=output_dtype_for(dtype), + dtype_acc="fp32", + layout_a="row", + layout_b="col", + layout_c="row", + tile_m=TILE_M, + tile_n=TILE_N, + tile_k=TILE_K, + wave_m=2, + wave_n=2, + wave_k=1, + warp_tile_m=32, + warp_tile_n=32, + warp_tile_k=16, + pipeline="compv3", + scheduler="intrawave", + epilogue="cshuffle", + pad_m=False, + pad_n=False, + pad_k=False, + persistent=False, + variant="stream_k", + reduction_strategy=strategy, + gfx_arch=gfx_arch, + ) + + +class _Shape: + """One shape: fp32 operands plus a per-dtype quantized reference. + + A and B are generated once and shared by every dtype/strategy; the fp32 + matmul is memoized per dtype so the 4096-cubed reference is computed four + times, not twelve. + """ + + def __init__(self, label: str, M: int, N: int, K: int, seed: int): + self.label = label + self.M, self.N, self.K = M, N, K + rng = np.random.default_rng(seed) + self.A = (rng.standard_normal((M, K)) * 0.1).astype(np.float32) + self.B = (rng.standard_normal((K, N)) * 0.1).astype(np.float32) + self.problem = GemmProblem(M=M, N=N, K=K) + self._refs = {} + + def reference(self, dtype: str) -> np.ndarray: + ref = self._refs.get(dtype) + if ref is None: + acc = _quantize(self.A, dtype) @ _quantize(self.B, dtype) + ref = _quantize(acc, output_dtype_for(dtype)) + self._refs[dtype] = ref + return ref + + +def _verify(label, so, shape, dtype, strategy): + """Build a runner for ``so``, launch once, and check the output.""" + runner = GpuGemmRunner(lib_path=so) + result = runner.run(shape.A, shape.B, shape.problem) + if not result.success: + return FAIL, f"{label}: launch failed (status={result.status})" + + got = np.asarray(result.output, dtype=np.float32) + if not np.all(np.isfinite(got)): + return FAIL, f"{label}: output contains NaN/Inf" + # An all-zero C is the signature of a Stream-K kernel that never landed its + # partials — the most likely failure mode for an unexercised strategy. + if not np.any(got): + return FAIL, f"{label}: output is all zeros" + + ref = shape.reference(dtype) + maxv = float(np.max(np.abs(ref))) + num_wgs = ( + S1_ATOMIC_WGS_PER_TILE if strategy == "atomic" else NON_ATOMIC_WGS_PER_TILE + ) + rtol, atol = _streamk_tolerance( + dtype, output_dtype_for(dtype), shape.K, num_wgs, maxv + ) + + bad = np.abs(got - ref) > atol + rtol * np.abs(ref) + rel = _max_rel(got, ref) + if bad.any(): + n_bad = int(bad.sum()) + return FAIL, ( + f"{label}: {n_bad}/{got.size} elements outside " + f"rtol={rtol:.3e} atol={atol:.3e} (max_rel={rel:.3e})" + ) + return PASS, ( + f"{label}: OK max_rel={rel:.3e} rtol={rtol:.3e} atol={atol:.3e} " + f"{result.time_ms:.3f} ms" + ) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def _build_matrix(dtypes): + """(label, dtype, strategy, shape_key) for every case we intend to run. + + S2 is deliberately linear/tree-only — see the module docstring. + """ + cases = [] + for dtype in dtypes: + for strategy in STRATEGIES: + cases.append((f"S1/{dtype}/{strategy}", dtype, strategy, "S1")) + for dtype in dtypes: + for strategy in STRATEGIES: + if strategy == "atomic": + continue + cases.append((f"S2/{dtype}/{strategy}", dtype, strategy, "S2")) + return cases + + +def main(): + parser = argparse.ArgumentParser( + description="Stream-K GPU correctness tests (atomic / linear / tree)" + ) + # No hardcoded default: it must stay possible to tell "user asked for gfx942" + # from "we are on an unrelated box", so the skip below can fire. + parser.add_argument("--gfx", default=None, + help="GPU arch override (default: auto-detect; gfx942 only)") + parser.add_argument("--dtypes", default=",".join(ALL_DTYPES), + help=f"comma-separated subset of {','.join(ALL_DTYPES)}") + parser.add_argument("-v", "--verbose", action="store_true") + args = parser.parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(levelname)s: %(message)s", + ) + + gfx = args.gfx or _detect_arch() + if not gfx: + print("SKIP: no supported GPU detected (rocminfo); Stream-K GPU tests skipped") + return SKIP_EXIT + if gfx not in SUPPORTED_ARCHS: + print(f"SKIP: Stream-K GPU tests are {'/'.join(SUPPORTED_ARCHS)}-only; " + f"detected {gfx}") + return SKIP_EXIT + + dtypes = [d.strip() for d in args.dtypes.split(",") if d.strip()] + unknown = [d for d in dtypes if d not in ALL_DTYPES] + if unknown: + print(f"ERROR: unknown dtype(s) {unknown}; choose from {list(ALL_DTYPES)}") + return 2 + + # bf16/fp8/bf8 host codecs come from ml_dtypes. fp16 needs nothing, so drop + # the others rather than skipping the whole suite. + try: + import ml_dtypes # noqa: F401 + except ImportError: + dropped = [d for d in dtypes if d != "fp16"] + if dropped: + log.warning("ml_dtypes not installed; dropping %s", ",".join(dropped)) + dtypes = [d for d in dtypes if d == "fp16"] + if not dtypes: + print("SKIP: no runnable dtypes (install ml_dtypes for bf16/fp8/bf8)") + return SKIP_EXIT + + # The default 50 warmup / 100 repeat iterations are pure waste for a + # correctness run; each launch here re-zeros and recomputes the full C. + os.environ.setdefault("CK_TILE_BENCH_WARMUP", "1") + os.environ.setdefault("CK_TILE_BENCH_REPEAT", "2") + + cases = _build_matrix(dtypes) + # One kernel per (dtype, strategy); S1 and S2 share it since M/N/K are + # runtime arguments, not codegen constants. + kernel_keys = sorted({(d, s) for _, d, s, _ in cases}) + configs = [_make_config(d, s, gfx) for d, s in kernel_keys] + + log.info("Building %d Stream-K kernels for %d cases ...", + len(configs), len(cases)) + sos = setup_multiple_gemm_dispatchers(configs, verbose=args.verbose) + by_key = dict(zip(kernel_keys, sos)) + + shapes = { + "S1": _Shape("S1", *S1, seed=42), + "S2": _Shape("S2", *S2, seed=43), + } + + results = [] + for label, dtype, strategy, shape_key in cases: + log.info("--- Running %s ---", label) + so = by_key.get((dtype, strategy)) + if so is None: + results.append((label, FAIL, f"{label}: codegen/compile failed")) + log.info("[%s] %s", FAIL, results[-1][2]) + continue + try: + status, detail = _verify( + label, so, shapes[shape_key], dtype, strategy + ) + except Exception as exc: + status, detail = FAIL, f"{label}: exception: {exc}" + results.append((label, status, detail)) + log.info("[%s] %s", status, detail) + + print("\n=== Summary ===") + passed = sum(1 for _, s, _ in results if s == PASS) + for name, status, detail in results: + print(f" [{status:4s}] {detail}") + print(f"\n{passed}/{len(results)} passed") + + return 0 if passed == len(results) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From f13b9eab7b646d9b983528b22119a3755295321f Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Tue, 25 Aug 2026 15:38:35 -0400 Subject: [PATCH 21/51] ci(ck-tile): register bridge GPU correctness tests in ctest and daily lanes Three script-style GPU tests were unreachable from ctest. test_bquant existed and was real (5 cases) but ran only from the Jenkins gfx950 lane; test_streamk and test_aquant are new. Each gets its own add_test + set_tests_properties pair with SKIP_RETURN_CODE 77 -- they cannot join the unittest-discover foreach block, which has no skip code and a fixed 600s timeout. Stream-K gets 900s for its 12 hipcc builds and 4096-cubed reference; the two quant tests get 600s. None is given --gfx. Each hipcc-compiles kernels with --offload-arch and then runs them, so the arch must match the PHYSICAL GPU; rocminfo auto-detection is the right source, and the CMake-configured GPU_TARGETS may legitimately differ from the card in the box. On the Jenkins side, test_streamk_gpu_correctness.py joins the gfx942 daily branch next to test_streamk_registry.py (registry-level and bridge-level coverage of the same operator), and test_aquant_gpu_correctness.py is appended last in the gfx950 branch. Appended last, deliberately: execute_cmd is a single &&-joined string, so the first non-zero exit aborts everything after it -- and a clean SKIP is exit 77, which is non-zero. A gfx950 test that skips silently truncates the rest of the lane. That hazard predates this change and is not fixed here, but new calls go at the end so a skip cannot mask a test that would otherwise have run, and a comment now records it. Co-Authored-By: Claude Opus 5 --- .../dispatcher/tests/CMakeLists.txt | 58 +++++++++++++++++++ .../composablekernel/groovy/vars/ck.groovy | 19 +++++- 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/projects/composablekernel/dispatcher/tests/CMakeLists.txt b/projects/composablekernel/dispatcher/tests/CMakeLists.txt index 9dab041ed790..070e0091cafe 100644 --- a/projects/composablekernel/dispatcher/tests/CMakeLists.txt +++ b/projects/composablekernel/dispatcher/tests/CMakeLists.txt @@ -245,6 +245,64 @@ set_tests_properties(dispatcher_test_streamk_registry PROPERTIES ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" ) +# GPU numeric-correctness tests for the quantized / Stream-K bridges. +# +# These are script-style (argparse + `return 77` to skip), not unittest, so each +# needs its own add_test + SKIP_RETURN_CODE pair -- they cannot join the +# `unittest discover` foreach above, which has no skip code and a fixed timeout. +# +# Deliberately NOT given a --gfx argument: each test hipcc-compiles kernels with +# --offload-arch and then runs them, so the arch must match the PHYSICAL GPU. +# rocminfo auto-detection is the right source for that; the CMake-configured +# GPU_TARGETS may legitimately differ from the card in the box. Every one of +# these exits 77 (Skipped, not Passed) when no GPU is present or the detected +# arch is not the one the operator supports. + +# Stream-K: atomic / linear / tree reduction through the ctypes bridge (gfx942). +# 12 hipcc builds plus a 4096-cubed reference, hence the larger timeout. +add_test( + NAME dispatcher_test_streamk_gpu_correctness + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_streamk_gpu_correctness.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/.. +) + +set_tests_properties(dispatcher_test_streamk_gpu_correctness PROPERTIES + LABELS "dispatcher;python;bridge;gpu" + TIMEOUT 900 + SKIP_RETURN_CODE 77 + ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" +) + +# AQuant grouped GEMM: fp8 / bf8 per-(m,k)-group A-side scales (gfx950). +add_test( + NAME dispatcher_test_aquant_gpu_correctness + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_aquant_gpu_correctness.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/.. +) + +set_tests_properties(dispatcher_test_aquant_gpu_correctness PROPERTIES + LABELS "dispatcher;python;bridge;gpu" + TIMEOUT 600 + SKIP_RETURN_CODE 77 + ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" +) + +# BQuant grouped GEMM: C4 (fp8/bf8) + H3 (mx_*) B-side scales (gfx950). +# The test already existed but ran only from the Jenkins gfx950 lane; this makes +# it reachable from ctest too. +add_test( + NAME dispatcher_test_bquant_gpu_correctness + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_bquant_gpu_correctness.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/.. +) + +set_tests_properties(dispatcher_test_bquant_gpu_correctness PROPERTIES + LABELS "dispatcher;python;bridge;gpu" + TIMEOUT 600 + SKIP_RETURN_CODE 77 + ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" +) + # Stress Test Script add_test( NAME dispatcher_stress_test diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 617684634b4b..974dc1a91bb6 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1639,9 +1639,16 @@ def runDispatcherCorrectnessTests(String arch, String compiler, String tier) { // Stream-K has its own registry-level driver test; its defaults are the // full 4x4 matrix (48 driver compiles / 96 GPU runs), so CI pins one // dtype/layout. Note it takes --arch, not --gfx. + // + // test_streamk_gpu_correctness.py is the bridge-level companion: the + // shared sweep above only ever builds the atomic reduction strategy + // (default_ci_config.json has no streamk_config, so expand_sweep falls + // back to ["atomic"]), leaving linear/tree unverified through ctypes. + // It takes --gfx and self-gates to gfx942. if (arch == "gfx942") { execute_cmd += """ && \ - python3 ../dispatcher/tests/test_streamk_registry.py --arch ${arch} --datatypes fp16 --layouts rcr""" + python3 ../dispatcher/tests/test_streamk_registry.py --arch ${arch} --datatypes fp16 --layouts rcr && \ + python3 ../dispatcher/tests/test_streamk_gpu_correctness.py --gfx ${arch}""" } // mx_gemm has no expand_sweep -- mx_gemm_utils exposes only // default_fp8_config()/default_fp4_config() -- so it cannot join the @@ -1655,10 +1662,18 @@ def runDispatcherCorrectnessTests(String arch, String compiler, String tier) { // yet, so invoking them here would fail the lane on "can't open file" // before reaching a GPU. Add the calls back in the same change that adds // the tests. + // + // CAUTION for anyone extending this chain: execute_cmd is a single + // &&-joined string, so the FIRST non-zero exit aborts everything after + // it -- and a clean SKIP is exit 77, which is non-zero. A gfx950 test + // that skips will therefore silently truncate the rest of the lane. + // Append new calls at the END so a skip cannot mask a test that would + // otherwise have run. (Pre-existing hazard, not introduced here.) if (arch == "gfx950") { execute_cmd += """ && \ python3 ../dispatcher/tests/test_bquant_gpu_correctness.py --gfx ${arch} && \ - python3 ../dispatcher/tests/test_mx_gemm_gpu_correctness.py""" + python3 ../dispatcher/tests/test_mx_gemm_gpu_correctness.py && \ + python3 ../dispatcher/tests/test_aquant_gpu_correctness.py --gfx ${arch}""" } } try { From c9e7c00c680a74b24f84e385c4b4c72093bfd9ba Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Tue, 25 Aug 2026 18:13:10 -0400 Subject: [PATCH 22/51] test(ck-tile): add GPU correctness test for ABQuant grouped GEMM grouped_gemm_abquant has a complete dispatcher bridge on develop but was the only operator in the bridge set with no GPU-side coverage -- only a CPU test_grouped_gemm_abquant_utils.py. Add test_abquant_gpu_correctness.py, modelled on its aquant sibling, with three cases: fp8/compv3, bf8/compv3 and fp8/eightwaves. The reference is ported from examples/gemm/python/15_grouped_gemm_abquant.py so that both operands are dequantised, which is the one real difference from aquant. Problem sizes are exact tile multiples because kPadM/kPadN/kPadK are false for every ABQuant prefill config: 256x256x512 spans a 2x2 grid of compv3's 128x128x128 tile, and 384x512x512 spans a 2x2 grid of eightwaves' 192x256x128. bquant_group_n follows the pipeline (1 for compv3, 128 for eightwaves). fp8 encoding is unconditionally OCP, which is correct here because ABQuant is gfx950-only and gfx950 compiles with -DCK_TILE_USE_OCP_FP8; the arch-aware codec is only needed for operators that also run on gfx942. Register it in ctest with SKIP_RETURN_CODE 77 and no --gfx argument, so the arch is autodetected and must match the physical GPU. Co-Authored-By: Claude Opus 5 --- .../dispatcher/tests/CMakeLists.txt | 15 + .../tests/test_abquant_gpu_correctness.py | 344 ++++++++++++++++++ 2 files changed, 359 insertions(+) create mode 100644 projects/composablekernel/dispatcher/tests/test_abquant_gpu_correctness.py diff --git a/projects/composablekernel/dispatcher/tests/CMakeLists.txt b/projects/composablekernel/dispatcher/tests/CMakeLists.txt index 83f41bdc0973..8d736df5cd28 100644 --- a/projects/composablekernel/dispatcher/tests/CMakeLists.txt +++ b/projects/composablekernel/dispatcher/tests/CMakeLists.txt @@ -345,6 +345,21 @@ set_tests_properties(dispatcher_test_bquant_gpu_correctness PROPERTIES ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" ) +# ABQuant grouped GEMM: fp8 / bf8 scales on BOTH operands, compv3 + the +# gfx950-native eightwaves pipeline (gfx950). +add_test( + NAME dispatcher_test_abquant_gpu_correctness + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_abquant_gpu_correctness.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/.. +) + +set_tests_properties(dispatcher_test_abquant_gpu_correctness PROPERTIES + LABELS "dispatcher;python;bridge;gpu" + TIMEOUT 600 + SKIP_RETURN_CODE 77 + ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" +) + # Stress Test Script add_test( NAME dispatcher_stress_test diff --git a/projects/composablekernel/dispatcher/tests/test_abquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_abquant_gpu_correctness.py new file mode 100644 index 000000000000..341ce4f43b0c --- /dev/null +++ b/projects/composablekernel/dispatcher/tests/test_abquant_gpu_correctness.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python3 + +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT + +""" +GPU correctness tests for ABQuant GEMM dispatcher — scales on *both* operands. + +Completes the quant trio alongside test_aquant_gpu_correctness.py (A-side only) +and test_bquant_gpu_correctness.py (B-side only). ABQuant scales A per (m, k) +group and B per (k, n) group in the same kernel: + + C = dequant(A, AQ) @ dequant(B, BQ) + A[m, k] *= AQ[m // aM, k // aK] + B[k, n] *= BQ[k // bK, n // bN] + +Requires a gfx950 GPU (MI350X / MI355X) and hipcc in PATH. Skips cleanly +(exit 77) when no GPU is visible or the detected arch is not gfx950, so it is +safe to invoke unconditionally from a CI lane. + +Tests: + fp8/compv3, bf8/compv3 — GPU output is non-zero, finite, and within 5% + max-relative-error vs. a fp32 CPU reference; time_ms is positive. + M=N=256 spans a 2x2 grid of 128x128 output tiles, so a constant + tile index cannot pass. + fp8/eightwaves — the gfx950-native 8-wave pipeline (192x256 tiles, + TransposeC=True, bquant_group_n=128), which exercises a different + MFMA path and B-scale granularity than compv3. + +Run: + python3 test_abquant_gpu_correctness.py + python3 test_abquant_gpu_correctness.py -v # verbose hipcc output + python3 test_abquant_gpu_correctness.py --gfx gfx950 +""" + +import argparse +import logging +import sys +import tempfile +from pathlib import Path + +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "python")) + +from grouped_gemm_abquant_utils import ( + ABQuantGemmProblem, + ABQuantGpuGemmRunner, + setup_multiple_abquant_dispatchers, + default_fp8_compv3_config, + default_bf8_compv3_config, + default_fp8_eightwaves_config, +) + +log = logging.getLogger(__name__) + +TOLERANCE = 0.05 # 5% max relative error — fp8/bf8 precision floor + +PASS = "PASS" +FAIL = "FAIL" + +# ctest reports this as "skipped" rather than passed. Without it, a CPU-only or +# non-gfx950 runner would report every ABQuant test as a hard FAIL for a reason +# that has nothing to do with the code under test. +SKIP_EXIT = 77 + +# ABQuant shares AQuant/BQuant's gfx950 (MI350X/MI355X) quantised-MFMA requirement. +SUPPORTED_ARCHS = ("gfx950",) + + +# --------------------------------------------------------------------------- +# Dtype helpers +# --------------------------------------------------------------------------- + +def _encode_fp8(arr: np.ndarray, dtype: str) -> np.ndarray: + """Encode float32 → fp8 bytes (uint8 view of the fp8 bit pattern). + + fp8 = OCP e4m3fn, bf8 = OCP e5m2. Unconditional OCP is correct here because + ABQuant is gfx950-only, and gfx950 kernels are compiled with + -DCK_TILE_USE_OCP_FP8 (see grouped_gemm_abquant_utils._compile_abquant_kernel). + Operators that also run on gfx942 need an arch-aware encoder instead, since + fp8_t there is the native FNUZ format. + """ + import ml_dtypes + + ml_t = ml_dtypes.float8_e4m3fn if dtype == "fp8" else ml_dtypes.float8_e5m2 + return arr.astype(ml_t).view(np.uint8) + + +def _decode_fp8(arr: np.ndarray, dtype: str) -> np.ndarray: + """Decode fp8 bytes (uint8 view) back to float32.""" + import ml_dtypes + + ml_t = ml_dtypes.float8_e4m3fn if dtype == "fp8" else ml_dtypes.float8_e5m2 + return arr.view(ml_t).astype(np.float32) + + +# --------------------------------------------------------------------------- +# CPU reference +# --------------------------------------------------------------------------- + +def _reference_gemm(A_f32: np.ndarray, B_f32: np.ndarray, + AQ: np.ndarray, BQ: np.ndarray, + problem: ABQuantGemmProblem) -> np.ndarray: + """C = dequant(A, AQ) @ dequant(B, BQ) in fp32. + + Mirrors reference_abquant_gemm in examples/gemm/python/15_grouped_gemm_abquant.py. + Unlike AQuant, both operands carry scales, and their group axes differ: + AQ is indexed (m, k) and BQ is indexed (k, n). + """ + M, N, K = problem.M, problem.N, problem.K + aM, aK = problem.aquant_group_m, problem.aquant_group_k + bK, bN = problem.bquant_group_k, problem.bquant_group_n + + A_dq = A_f32.astype(np.float32).copy() + for qi in range(problem.QM_A): + for qk in range(problem.QK_A): + m0, m1 = qi * aM, min((qi + 1) * aM, M) + k0, k1 = qk * aK, min((qk + 1) * aK, K) + A_dq[m0:m1, k0:k1] *= float(AQ[qi, qk]) + + B_dq = B_f32.astype(np.float32).copy() + for qk in range(problem.QK_B): + for qn in range(problem.QN_B): + k0, k1 = qk * bK, min((qk + 1) * bK, K) + n0, n1 = qn * bN, min((qn + 1) * bN, N) + B_dq[k0:k1, n0:n1] *= float(BQ[qk, qn]) + + return (A_dq @ B_dq).astype(np.float32) + + +def _max_rel_err(C_gpu: np.ndarray, C_ref: np.ndarray) -> float: + C_gpu_f = C_gpu.astype(np.float32) + C_ref_f = C_ref.astype(np.float32) + num = np.abs(C_gpu_f - C_ref_f) + # Use 1% of the global max magnitude as the denominator floor to avoid + # inflating the relative error when individual elements are near zero + # (a common occurrence with random inputs that partially cancel in GEMM). + ref_max = float(np.abs(C_ref_f).max()) + den = np.abs(C_ref_f) + max(ref_max * 1e-2, 1e-6) + return float(np.max(num / den)) + + +# --------------------------------------------------------------------------- +# Core test helper +# --------------------------------------------------------------------------- + +def _detect_arch() -> "str | None": + """Detected gfx arch, or None when no GPU is visible. + + The empty fallback keeps absence detectable; detect_gpu_arch would otherwise + invent a default and turn "no GPU" into a confusing build failure. + """ + try: + from dispatcher_common import detect_gpu_arch + + return detect_gpu_arch(fallback="") or None + except Exception: + return None + + +def _make_inputs(problem: ABQuantGemmProblem, dtype: str, seed: int = 42): + """Generate fp8-encoded A/B plus float32 AQ/BQ scales, and the decoded references. + + A and B are encoded to fp8 and decoded back so the CPU reference consumes the + same rounded values the kernel does; otherwise the comparison would be + dominated by quantisation error rather than kernel correctness. + + AQ is row-major [QM_A, QK_A]; BQ is column-major [QK_B, QN_B] — the runner + applies the Fortran-order view, so the shape here stays logical. + """ + rng = np.random.default_rng(seed) + M, N, K = problem.M, problem.N, problem.K + A_f32 = rng.uniform(-2.0, 2.0, (M, K)).astype(np.float32) + B_f32 = rng.uniform(-2.0, 2.0, (K, N)).astype(np.float32) + AQ = rng.uniform(0.5, 2.0, (problem.QM_A, problem.QK_A)).astype(np.float32) + BQ = rng.uniform(0.5, 2.0, (problem.QK_B, problem.QN_B)).astype(np.float32) + A_raw = _encode_fp8(A_f32, dtype) + B_raw = _encode_fp8(B_f32, dtype) + return A_raw, _decode_fp8(A_raw, dtype), B_raw, _decode_fp8(B_raw, dtype), AQ, BQ + + +def _run_one(label: str, config, M: int, N: int, K: int, dtype: str, + out_dir: Path, gfx_arch: str = "gfx950", + seed: int = 42) -> "tuple[str, str]": + """Build, run, and verify one kernel. Returns (PASS|FAIL, detail_message).""" + # The problem must repeat the config's quant grouping: the kernel bakes the + # group sizes into the generated header, and the host strides are derived + # from them, so a mismatch silently reads the wrong scale. + problem = ABQuantGemmProblem( + M=M, N=N, K=K, + aquant_group_m=config.aquant_group_m, + aquant_group_n=config.aquant_group_n, + aquant_group_k=config.aquant_group_k, + bquant_group_m=config.bquant_group_m, + bquant_group_n=config.bquant_group_n, + bquant_group_k=config.bquant_group_k, + ) + + so_paths = setup_multiple_abquant_dispatchers( + configs=[config], + output_dir=out_dir, + gfx_arch=gfx_arch, + ) + if not so_paths or so_paths[0] is None: + return FAIL, f"{label}: kernel build failed" + + A_raw, A_dec, B_raw, B_dec, AQ, BQ = _make_inputs(problem, dtype, seed=seed) + + runner = ABQuantGpuGemmRunner(so_paths[0]) + + result = runner.run(A=A_raw, B=B_raw, AQ=AQ, BQ=BQ, problem=problem, + c_dtype=np.float16) + C_gpu = result.C.astype(np.float32) + + # An all-zero C is the signature of a kernel that launched but never wrote — + # e.g. an MFMA instruction unavailable on the target arch. + if np.all(C_gpu == 0): + return FAIL, f"{label}: GPU output is all-zero" + nan_mask = ~np.isfinite(C_gpu) + if np.any(nan_mask): + nan_frac = nan_mask.mean() + log.debug("%s: NaN/Inf fraction=%.3f, first 8 elements=%s", + label, nan_frac, C_gpu.flat[:8].tolist()) + return FAIL, f"{label}: GPU output contains NaN/Inf (frac={nan_frac:.3f})" + + C_ref = _reference_gemm(A_dec, B_dec, AQ, BQ, problem) + mre = _max_rel_err(C_gpu, C_ref) + if mre > TOLERANCE: + return FAIL, (f"{label}: max_rel_err={mre:.4f} > tol={TOLERANCE:.4f} " + f"(shape M={M} N={N} K={K})") + + # Timing sanity: a zero time_ms means the HIP event pair never bracketed a launch. + result_timed = runner.run(A=A_raw, B=B_raw, AQ=AQ, BQ=BQ, problem=problem, + c_dtype=np.float16) + if result_timed.time_ms <= 0.0: + return FAIL, f"{label}: time_ms={result_timed.time_ms:.4f} is not positive" + + return PASS, (f"{label}: max_rel_err={mre:.4f}, " + f"time_ms={result_timed.time_ms:.3f}") + + +# --------------------------------------------------------------------------- +# Individual test cases +# --------------------------------------------------------------------------- +# +# kPadM/kPadN/kPadK are false for every ABQuant prefill config, so M, N and K +# must be exact multiples of TileM, TileN and TileK. Shapes below are the +# smallest multi-tile sizes that satisfy that for each pipeline. + +def test_fp8_compv3(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": + # compv3 tiles are 128x128x128; M=N=256 spans a 2x2 tile grid. + # bquant_group_n=1 gives one B scale per column, the finest granularity. + M, N, K, gK, bN = 256, 256, 512, 128, 1 + cfg = default_fp8_compv3_config(quant_group_k=gK, bquant_group_n=bN, + gfx_arch=gfx_arch) + return _run_one("fp8/compv3", cfg, M, N, K, "fp8", out_dir, + gfx_arch=gfx_arch, seed=42) + + +def test_bf8_compv3(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": + M, N, K, gK, bN = 256, 256, 512, 128, 1 + cfg = default_bf8_compv3_config(quant_group_k=gK, bquant_group_n=bN, + gfx_arch=gfx_arch) + return _run_one("bf8/compv3", cfg, M, N, K, "bf8", out_dir, + gfx_arch=gfx_arch, seed=43) + + +def test_fp8_eightwaves(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": + # eightwaves tiles are 192x256x128, so M must be a multiple of 192 and N of + # 256. bquant_group_n=128 is the only granularity the C++ tests validate for + # this pipeline. + M, N, K, gK, bN = 384, 512, 512, 128, 128 + cfg = default_fp8_eightwaves_config(quant_group_k=gK, bquant_group_n=bN, + gfx_arch=gfx_arch) + return _run_one("fp8/eightwaves", cfg, M, N, K, "fp8", out_dir, + gfx_arch=gfx_arch, seed=44) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +TESTS = [ + ("fp8/compv3", test_fp8_compv3), + ("bf8/compv3", test_bf8_compv3), + ("fp8/eightwaves", test_fp8_eightwaves), +] + + +def main(): + parser = argparse.ArgumentParser(description="ABQuant GPU correctness tests") + # No hardcoded default: it must stay possible to tell "user asked for + # gfx950" from "we are on an unrelated box", so the skip below can fire. + parser.add_argument("--gfx", default=None, + help="GPU arch override (default: auto-detect; gfx950 only)") + parser.add_argument("-v", "--verbose", action="store_true") + parser.add_argument("--output-dir", type=Path, default=None) + args = parser.parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(levelname)s: %(message)s", + ) + + gfx = args.gfx or _detect_arch() + if not gfx: + print("SKIP: no supported GPU detected (rocminfo); ABQuant GPU tests skipped") + return SKIP_EXIT + if gfx not in SUPPORTED_ARCHS: + print(f"SKIP: ABQuant is {'/'.join(SUPPORTED_ARCHS)}-only; detected {gfx}") + return SKIP_EXIT + try: + import ml_dtypes # noqa: F401 + except ImportError: + # Without ml_dtypes there is no trustworthy fp8 codec here, and a + # stand-in would compare the kernel against the wrong values. + print("SKIP: ml_dtypes not installed; ABQuant fp8/bf8 encoding unavailable") + return SKIP_EXIT + + out_dir = args.output_dir or Path(tempfile.mkdtemp(prefix="abquant_gpu_test_")) + log.info("Kernel output dir: %s", out_dir) + + results = [] + for name, fn in TESTS: + log.info("--- Running %s ---", name) + try: + status, detail = fn(out_dir, gfx) + except Exception as exc: + status, detail = FAIL, f"{name}: exception: {exc}" + results.append((name, status, detail)) + log.info("[%s] %s", status, detail) + + print("\n=== Summary ===") + passed = sum(1 for _, s, _ in results if s == PASS) + for name, status, detail in results: + print(f" [{status:4s}] {detail}") + print(f"\n{passed}/{len(results)} passed") + + return 0 if passed == len(results) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From 8d9c849f2922e2abae0321a5586cc195210d0d98 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Tue, 25 Aug 2026 18:14:19 -0400 Subject: [PATCH 23/51] ci(ck-tile): extend dispatcher smoke lane to all 11 develop operators RUN_DISPATCHER_CORRECTNESS_TESTS verified only four things on gfx942 -- the gemm_universal sweep, parity, batched_gemm and batched_contraction. Every other dispatcher operator was reachable only through the _DAILY sibling, which is cron-only on develop, so a developer touching the bridge got no pre-merge signal on grouped, multi_d, multi_abd, stream_k or any of the quant ops. Un-gate the operator chain from the tier. Both tiers now run the same per-arch operator set and the tier controls cost only: the gemm_universal sweep stays at 64 for smoke and 500 for daily, and the per-variant sweeps gain their own budget, 16 for smoke against 64 for daily. Invoke the dedicated grouped/multi_d/multi_abd GPU tests, which were registered in ctest but never called from here. The --variant sweep covers their search space; these cover the ctypes bridge. Add a run_ok() helper. execute_cmd is one &&-joined string and a clean skip is exit 77, so a skipping test silently killed everything after it. The gfx950 block goes from three chained script-style tests to four, so appending at the end no longer suffices. Only 77 is swallowed; any other non-zero exit still fails the lane. The quant operators pin _DEFAULT_GFX_ARCH = "gfx950", so the full operator surface cannot be covered from one node type. Give the smoke stage a gfx950 branch, reversing the earlier gfx942-only decision, and refresh both param descriptions -- the _DAILY one still advertised rowcolquant and tensorquant, which this lane has never run. Those two stay out for now. #10010 has since landed their bridges and GPU tests on develop, so they are unblocked, but neither test self-gates on the detected arch the way its neighbours do, so wiring them in is a follow-up. Expect the smoke lane's wall clock to rise: it now books a gfx950 node and JIT-compiles a .so set per variant. If that proves too slow, the cheapest lever is dropping the three dedicated bridge tests from smoke. Co-Authored-By: Claude Opus 5 --- projects/composablekernel/Jenkinsfile | 33 ++++- .../composablekernel/groovy/vars/ck.groovy | 135 ++++++++++-------- 2 files changed, 105 insertions(+), 63 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index 20191fa1ad86..4789e9d88aed 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -146,11 +146,11 @@ pipeline { booleanParam( name: "RUN_DISPATCHER_CORRECTNESS_TESTS", defaultValue: false, - description: "Smoke-tier verification of JIT-compiled dispatcher kernels on gfx942: gemm_universal sweep at budget 64 plus parity, batched_gemm and batched_contraction (default: OFF)") + description: "Smoke-tier verification of JIT-compiled dispatcher kernels on gfx942 + gfx950: gemm_universal sweep at budget 64 plus parity, grouped, multi_d, multi_abd, batched_gemm, batched_contraction, stream_k on gfx942, and mx_gemm + aquant + bquant + abquant on gfx950 (per-arch subset; default: OFF)") booleanParam( name: "RUN_DISPATCHER_CORRECTNESS_TESTS_DAILY", defaultValue: false, - description: "Full-tier verification of JIT-compiled dispatcher kernels on gfx942 + gfx950: gemm_universal sweep at budget 500 plus grouped, multi_d, multi_abd, batched_gemm, batched_contraction, stream_k on gfx942, and bquant + mx_gemm + rowcolquant + tensorquant on gfx950 (per-arch subset; default: OFF)") + description: "Full-tier verification of JIT-compiled dispatcher kernels on gfx942 + gfx950: the same operator set as the smoke tier, at budget 500 for the gemm_universal sweep and 64 for each variant sweep (default: OFF)") booleanParam( name: "BUILD_INSTANCES_ONLY", defaultValue: false, @@ -744,9 +744,11 @@ pipeline { } } } - // Smoke tier is gfx942-only on purpose: gfx950 is where the quant ops and - // the OCP-vs-FNUZ fp8 split live, which is full-tier territory, and a - // smoke lane that books two GPU node types is no longer a smoke lane. + // Both tiers fan out to gfx942 and gfx950. The quant operators (mx_gemm, + // aquant, bquant, abquant) are gfx950-only, so covering the full operator + // surface is impossible from a single node type. What separates smoke from + // daily is the search budget, not the operator count -- see the tier table + // on runDispatcherCorrectnessTests in groovy/vars/ck.groovy. stage("Run DISPATCHER_CORRECTNESS Tests") { when { @@ -773,6 +775,27 @@ pipeline { } } } + stage("Run DISPATCHER_CORRECTNESS Tests on gfx950") + { + when { + beforeAgent true + expression { params.RUN_DISPATCHER_CORRECTNESS_TESTS.toBoolean() } + } + agent none + steps{ + script { + loadCk() + ck.runOnHealthyNode(rocmnode("gfx950")) { + deleteDir() + ck.runDispatcherCorrectnessTests("gfx950", params.BUILD_COMPILER, "smoke") + cleanWs() + } + } + } + } + // No gfx1201 dispatcher stage: the dispatcher has no RDNA4 tile + // coverage, so such a stage could only ever run red. See the + // arch note on dispatcherGemmVariantsFor in groovy/vars/ck.groovy. } } stage("Run DISPATCHER_CORRECTNESS_DAILY Tests") diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 3f0abaacd95e..76303e27f5ef 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1582,36 +1582,52 @@ def dispatcherGemmVariantsFor(String arch) { // dtypes/layouts are deliberately omitted: the runner picks per-variant defaults // (multi_d/multi_abd are fp16/rcrr-only, so naming the standard 4x4 matrix here // would enumerate configs that variant cannot build). multi_d likewise takes the -// runner's default --elementwise-op PassThrough. The budget is well under the -// standard lane's 500 because each variant JIT-compiles its own .so set and -// these run in addition to, not instead of, the standard sweep. +// runner's default --elementwise-op PassThrough. +// +// Both tiers run every variant, so the budget is what separates them. It stays +// far under the standard sweep's (500 daily / 64 smoke) because each variant +// JIT-compiles its own .so set and these run in addition to, not instead of, +// that sweep -- 16 is enough for smoke to prove each variant still builds and +// matches its reference, without paying the daily lane's coverage. def dispatcherVariantCmd(String arch, String variant, String tier) { + def budget = (tier == "daily") ? 64 : 16 return """python3 ../dispatcher/tests/test_gemm_search_space.py \ --variant ${variant} \ --arch ${arch} \ - --budget 64 \ + --budget ${budget} \ --warmup 5 \ --repeat 5 \ --size 1024 \ --json dispatcher_${variant}_results_${tier}.json""" } -// Verify JIT-compiled dispatcher kernels against a host reference. Two tiers: +// Verify JIT-compiled dispatcher kernels against a host reference. +// +// BOTH tiers run the same operator set for the arch -- every dispatcher operator +// with a bridge on develop. A developer touching the bridge needs pre-merge +// signal on all of them, and a lane that covers four operators cannot give that. +// The tier controls cost only: +// +// "smoke" -- standard sweep at --budget 64, variants at --budget 16. +// "daily" -- standard sweep at --budget 500, variants at --budget 64. // -// "smoke" -- the standard sweep at --budget 64 plus the three fixed-config -// tests (parity, batched_gemm, batched_contraction). The dtype and -// layout lists stay at the full 4x4 on purpose: the sweep samples -// per stratum, so 64 spreads ~4 configs across all 16 dtype x -// layout combinations rather than testing one combination deeply. -// "daily" -- the same, at --budget 500, plus every non-standard GEMM variant -// this arch can run and the arch-specific extras below. +// The dtype and layout lists stay at the full 4x4 in both tiers on purpose: the +// sweep samples per stratum, so even 64 spreads ~4 configs across all 16 dtype x +// layout combinations rather than testing one combination deeply. // // As in runDispatcherPerfTests, result filenames carry the tier so a build with // both lanes enabled does not have one stage's artifacts clobber the other's. def runDispatcherCorrectnessTests(String arch, String compiler, String tier) { def daily = (tier == "daily") def budget = daily ? 500 : 64 + // run_ok wraps the script-style tests, which signal a clean skip with exit + // 77. execute_cmd is one &&-joined string, so without this an unsupported + // arch or a missing ml_dtypes would abort the lane at that point and every + // later operator would silently never run. Only 77 is swallowed; any other + // non-zero exit still fails the lane. A function definition itself exits 0, + // so it chains cleanly ahead of cmake. def execute_cmd = """ + run_ok() { "\$@"; rc=\$?; if [ \$rc -eq 77 ]; then echo "SKIP(77): \$*"; return 0; fi; return \$rc; } && \ cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ -D CMAKE_CXX_COMPILER="${compiler}" \ -D CMAKE_BUILD_TYPE=Release \ @@ -1630,53 +1646,56 @@ def runDispatcherCorrectnessTests(String arch, String compiler, String tier) { --size 1024 \ --json dispatcher_gemm_results_${tier}.json && \ python3 ../dispatcher/tests/test_gemm_parity.py && \ - python3 ../dispatcher/tests/test_batched_gemm_gpu_correctness.py --gfx ${arch} && \ - python3 ../dispatcher/tests/test_batched_contraction_gpu_correctness.py --gfx ${arch}""" - // Everything below is daily-only: each variant JIT-compiles its own .so set, - // which is the bulk of this lane's wall clock and the opposite of a smoke run. - if (daily) { - dispatcherGemmVariantsFor(arch).each { variant -> - execute_cmd += " && \\\n " + dispatcherVariantCmd(arch, variant, tier) - } - // Stream-K has its own registry-level driver test; its defaults are the - // full 4x4 matrix (48 driver compiles / 96 GPU runs), so CI pins one - // dtype/layout. Note it takes --arch, not --gfx. - // - // test_streamk_gpu_correctness.py is the bridge-level companion: the - // shared sweep above only ever builds the atomic reduction strategy - // (default_ci_config.json has no streamk_config, so expand_sweep falls - // back to ["atomic"]), leaving linear/tree unverified through ctypes. - // It takes --gfx and self-gates to gfx942. - if (arch == "gfx942") { - execute_cmd += """ && \ - python3 ../dispatcher/tests/test_streamk_registry.py --arch ${arch} --datatypes fp16 --layouts rcr && \ - python3 ../dispatcher/tests/test_streamk_gpu_correctness.py --gfx ${arch}""" - } - // mx_gemm has no expand_sweep -- mx_gemm_utils exposes only - // default_fp8_config()/default_fp4_config() -- so it cannot join the - // --variant sweep above and instead runs the two-config smoke test merged - // in #10132. That test is unittest-based and self-gates to gfx950, so it - // takes no --gfx (unlike test_bquant, which does). - // - // rowcolquant and tensorquant are deliberately absent: the tile_engine - // side of both ops exists, but dispatcher/tests carries no - // test_rowcolquant_gpu_correctness.py / test_tensorquant_gpu_correctness.py - // yet, so invoking them here would fail the lane on "can't open file" - // before reaching a GPU. Add the calls back in the same change that adds - // the tests. - // - // CAUTION for anyone extending this chain: execute_cmd is a single - // &&-joined string, so the FIRST non-zero exit aborts everything after - // it -- and a clean SKIP is exit 77, which is non-zero. A gfx950 test - // that skips will therefore silently truncate the rest of the lane. - // Append new calls at the END so a skip cannot mask a test that would - // otherwise have run. (Pre-existing hazard, not introduced here.) - if (arch == "gfx950") { - execute_cmd += """ && \ - python3 ../dispatcher/tests/test_bquant_gpu_correctness.py --gfx ${arch} && \ + run_ok python3 ../dispatcher/tests/test_batched_gemm_gpu_correctness.py --gfx ${arch} && \ + run_ok python3 ../dispatcher/tests/test_batched_contraction_gpu_correctness.py --gfx ${arch} && \ + python3 ../dispatcher/tests/test_grouped_gemm_gpu_correctness.py && \ + python3 ../dispatcher/tests/test_multi_d_gpu_correctness.py && \ + python3 ../dispatcher/tests/test_multi_abd_gpu_correctness.py""" + // The three tests just added are the bridge-level companions to the grouped + // /multi_d/multi_abd --variant sweeps below: the sweep exercises the search + // space, they exercise the ctypes bridge. They were registered in ctest but + // never invoked from here. All three are unittest-based, take no --gfx, and + // exit 0 on an internal skip, so they need no run_ok. + dispatcherGemmVariantsFor(arch).each { variant -> + execute_cmd += " && \\\n " + dispatcherVariantCmd(arch, variant, tier) + } + // Stream-K has its own registry-level driver test; its defaults are the + // full 4x4 matrix (48 driver compiles / 96 GPU runs), so CI pins one + // dtype/layout. Note it takes --arch, not --gfx. + // + // test_streamk_gpu_correctness.py is the bridge-level companion: the + // shared sweep above only ever builds the atomic reduction strategy + // (default_ci_config.json has no streamk_config, so expand_sweep falls + // back to ["atomic"]), leaving linear/tree unverified through ctypes. + // It takes --gfx and self-gates to gfx942. + if (arch == "gfx942") { + execute_cmd += """ && \ + run_ok python3 ../dispatcher/tests/test_streamk_registry.py --arch ${arch} --datatypes fp16 --layouts rcr && \ + run_ok python3 ../dispatcher/tests/test_streamk_gpu_correctness.py --gfx ${arch}""" + } + // The block-scale quant operators are gfx950-only: their utils modules pin + // _DEFAULT_GFX_ARCH = "gfx950" and the tests self-gate to it. This is why the + // smoke tier fans out to a gfx950 node as well as gfx942 -- see the stage + // comment in the Jenkinsfile. + // + // mx_gemm has no expand_sweep -- mx_gemm_utils exposes only + // default_fp8_config()/default_fp4_config() -- so it cannot join the + // --variant sweep above and instead runs the two-config smoke test merged + // in #10132. That test is unittest-based and self-gates to gfx950, so it + // takes no --gfx and needs no run_ok (unlike its neighbours, which do). + // + // rowcolquant and tensorquant are absent but no longer blocked: #10010 + // landed their dispatcher bridges and ctest-registered GPU tests on develop + // after this lane's operator set was chosen. Wiring them in is a follow-up, + // not an oversight -- unlike their neighbours here, neither test self-gates + // on the detected arch (--gfx just defaults to gfx950), so adding them needs + // that skip path checked first or an unsupported node will run them red. + if (arch == "gfx950") { + execute_cmd += """ && \ + run_ok python3 ../dispatcher/tests/test_bquant_gpu_correctness.py --gfx ${arch} && \ python3 ../dispatcher/tests/test_mx_gemm_gpu_correctness.py && \ - python3 ../dispatcher/tests/test_aquant_gpu_correctness.py --gfx ${arch}""" - } + run_ok python3 ../dispatcher/tests/test_aquant_gpu_correctness.py --gfx ${arch} && \ + run_ok python3 ../dispatcher/tests/test_abquant_gpu_correctness.py --gfx ${arch}""" } try { buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) From 1d17f742df64707e104cad0d07e34e17f835289c Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Tue, 25 Aug 2026 18:34:33 -0400 Subject: [PATCH 24/51] ci(ck-tile): add rowcolquant and tensorquant to the dispatcher smoke lane These were the two operators cut from the requested set when this lane was widened, on the grounds that neither had a dispatcher bridge on develop. That is no longer true: #10010 landed both bridges along with their ctest-registered GPU tests, so the exclusion no longer has a basis. Adding them brings the lane to the full 13-operator surface. Both go in the arch-independent head of the chain rather than the gfx950 block. Unlike aquant/bquant/abquant/mx_gemm, these two support gfx942 too -- _SUPPORTED_ARCHES is ("gfx942", "gfx950"), and _fp8_uses_ocp mirrors the FNUZ/OCP split instead of assuming OCP -- so pinning them to gfx950 would throw away half their coverage. Neither test could be dropped into the chain as-is. --gfx defaulted to a hardcoded "gfx950" with no autodetect, so on any other node they would compile for the wrong arch and crash rather than skip, taking the rest of the && chain down with them. main() now autodetects when --gfx is absent (keeping the explicit flag as a caller-trusted override, matching every sibling test) and returns SKIP_EXIT 77 for: no detected GPU, an unsupported arch, missing hipcc, or missing ml_dtypes. The lane's run_ok helper and ctest's SKIP_RETURN_CODE both read 77 as a clean skip. The ml_dtypes check is deliberately up front rather than delegated to _require_ml_dtypes: that helper raises pytest.skip's Skipped, which derives from BaseException and would slip past the per-test except Exception. ctest needs no change -- it drives both through -m pytest, where the requires_gpu marker already skips and exits 0. Only the standalone main() that Jenkins calls needed the gate. Co-Authored-By: Claude Opus 5 --- projects/composablekernel/Jenkinsfile | 2 +- .../tests/test_rowcolquant_gpu_correctness.py | 31 +++++++++++++++++-- .../tests/test_tensorquant_gpu_correctness.py | 31 +++++++++++++++++-- .../composablekernel/groovy/vars/ck.groovy | 30 +++++++++--------- 4 files changed, 74 insertions(+), 20 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index 4789e9d88aed..ef97e92f4fc6 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -146,7 +146,7 @@ pipeline { booleanParam( name: "RUN_DISPATCHER_CORRECTNESS_TESTS", defaultValue: false, - description: "Smoke-tier verification of JIT-compiled dispatcher kernels on gfx942 + gfx950: gemm_universal sweep at budget 64 plus parity, grouped, multi_d, multi_abd, batched_gemm, batched_contraction, stream_k on gfx942, and mx_gemm + aquant + bquant + abquant on gfx950 (per-arch subset; default: OFF)") + description: "Smoke-tier verification of JIT-compiled dispatcher kernels on gfx942 + gfx950: gemm_universal sweep at budget 64 plus parity, grouped, multi_d, multi_abd, batched_gemm, batched_contraction, rowcolquant, tensorquant on both arches, stream_k on gfx942, and mx_gemm + aquant + bquant + abquant on gfx950 (per-arch subset; default: OFF)") booleanParam( name: "RUN_DISPATCHER_CORRECTNESS_TESTS_DAILY", defaultValue: false, diff --git a/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py index 07a6c49df8f9..b487276f46d0 100644 --- a/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py @@ -43,6 +43,7 @@ log = logging.getLogger(__name__) TOLERANCE = 0.05 # 5% max relative error — fp8/bf8 precision floor +SKIP_EXIT = 77 # ctest SKIP_RETURN_CODE; see main() # --------------------------------------------------------------------------- @@ -298,7 +299,12 @@ def test_rowcolquant_timing_positive(tmp_path): def main(): parser = argparse.ArgumentParser( description="RowColQuant GPU correctness tests") - parser.add_argument("--gfx", default="gfx950") + # No hardcoded default: an explicit --gfx is an override the caller is + # trusted on, but falling back to a fixed arch would make a CPU-only or + # gfx90a box compile for gfx950 and crash instead of skipping. + parser.add_argument("--gfx", default=None, + help=f"GPU arch override (default: auto-detect; " + f"{'/'.join(_SUPPORTED_ARCHES)} only)") parser.add_argument("-v", "--verbose", action="store_true") parser.add_argument("--output-dir", type=Path, default=None) args = parser.parse_args() @@ -308,6 +314,27 @@ def main(): format="%(levelname)s: %(message)s", ) + # The pytest path gates on the requires_gpu marker; this standalone path is + # what CI drives, so it needs the same gates spelled out. SKIP_EXIT keeps a + # clean skip distinguishable from a failure -- ctest maps it via + # SKIP_RETURN_CODE and the Jenkins lane via its run_ok helper. + gfx = args.gfx or _GFX_ARCH + if not gfx: + print("SKIP: no supported GPU detected (rocm_agent_enumerator)") + return SKIP_EXIT + if gfx not in _SUPPORTED_ARCHES: + print(f"SKIP: RowColQuant needs native fp8 " + f"({'/'.join(_SUPPORTED_ARCHES)}); detected {gfx}") + return SKIP_EXIT + if not _has_hipcc(): + print("SKIP: hipcc not found in PATH; cannot JIT-compile kernels") + return SKIP_EXIT + if _ml_dtypes is None: + # _require_ml_dtypes raises pytest.skip's Skipped, which derives from + # BaseException and so slips past the per-test except Exception below. + print("SKIP: ml_dtypes not installed; fp8/bf8 encoding unavailable") + return SKIP_EXIT + out_dir = args.output_dir or Path(tempfile.mkdtemp(prefix="rowcolquant_gpu_test_")) log.info("Kernel output dir: %s", out_dir) @@ -315,7 +342,7 @@ def main(): for name, fn in TESTS: log.info("--- Running %s ---", name) try: - status, detail = fn(out_dir, args.gfx) + status, detail = fn(out_dir, gfx) except Exception as exc: status, detail = "FAIL", f"{name}: exception: {exc}" results.append((name, status, detail)) diff --git a/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py index 0fbaa9d8221d..dcd54512ae31 100644 --- a/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py @@ -43,6 +43,7 @@ log = logging.getLogger(__name__) TOLERANCE = 0.05 # 5% max relative error — fp8/bf8 precision floor +SKIP_EXIT = 77 # ctest SKIP_RETURN_CODE; see main() # --------------------------------------------------------------------------- @@ -296,7 +297,12 @@ def test_tensorquant_timing_positive(tmp_path): def main(): parser = argparse.ArgumentParser( description="TensorQuant GPU correctness tests") - parser.add_argument("--gfx", default="gfx950") + # No hardcoded default: an explicit --gfx is an override the caller is + # trusted on, but falling back to a fixed arch would make a CPU-only or + # gfx90a box compile for gfx950 and crash instead of skipping. + parser.add_argument("--gfx", default=None, + help=f"GPU arch override (default: auto-detect; " + f"{'/'.join(_SUPPORTED_ARCHES)} only)") parser.add_argument("-v", "--verbose", action="store_true") parser.add_argument("--output-dir", type=Path, default=None) args = parser.parse_args() @@ -306,6 +312,27 @@ def main(): format="%(levelname)s: %(message)s", ) + # The pytest path gates on the requires_gpu marker; this standalone path is + # what CI drives, so it needs the same gates spelled out. SKIP_EXIT keeps a + # clean skip distinguishable from a failure -- ctest maps it via + # SKIP_RETURN_CODE and the Jenkins lane via its run_ok helper. + gfx = args.gfx or _GFX_ARCH + if not gfx: + print("SKIP: no supported GPU detected (rocm_agent_enumerator)") + return SKIP_EXIT + if gfx not in _SUPPORTED_ARCHES: + print(f"SKIP: TensorQuant needs native fp8 " + f"({'/'.join(_SUPPORTED_ARCHES)}); detected {gfx}") + return SKIP_EXIT + if not _has_hipcc(): + print("SKIP: hipcc not found in PATH; cannot JIT-compile kernels") + return SKIP_EXIT + if _ml_dtypes is None: + # _require_ml_dtypes raises pytest.skip's Skipped, which derives from + # BaseException and so slips past the per-test except Exception below. + print("SKIP: ml_dtypes not installed; fp8/bf8 encoding unavailable") + return SKIP_EXIT + out_dir = args.output_dir or Path(tempfile.mkdtemp(prefix="tensorquant_gpu_test_")) log.info("Kernel output dir: %s", out_dir) @@ -313,7 +340,7 @@ def main(): for name, fn in TESTS: log.info("--- Running %s ---", name) try: - status, detail = fn(out_dir, args.gfx) + status, detail = fn(out_dir, gfx) except Exception as exc: status, detail = "FAIL", f"{name}: exception: {exc}" results.append((name, status, detail)) diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 76303e27f5ef..c4684658798f 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1604,8 +1604,8 @@ def dispatcherVariantCmd(String arch, String variant, String tier) { // Verify JIT-compiled dispatcher kernels against a host reference. // // BOTH tiers run the same operator set for the arch -- every dispatcher operator -// with a bridge on develop. A developer touching the bridge needs pre-merge -// signal on all of them, and a lane that covers four operators cannot give that. +// with a bridge on develop -- 13 of them. A developer touching the bridge needs +// pre-merge signal on all of them, and a lane covering four cannot give that. // The tier controls cost only: // // "smoke" -- standard sweep at --budget 64, variants at --budget 16. @@ -1650,12 +1650,19 @@ def runDispatcherCorrectnessTests(String arch, String compiler, String tier) { run_ok python3 ../dispatcher/tests/test_batched_contraction_gpu_correctness.py --gfx ${arch} && \ python3 ../dispatcher/tests/test_grouped_gemm_gpu_correctness.py && \ python3 ../dispatcher/tests/test_multi_d_gpu_correctness.py && \ - python3 ../dispatcher/tests/test_multi_abd_gpu_correctness.py""" - // The three tests just added are the bridge-level companions to the grouped - // /multi_d/multi_abd --variant sweeps below: the sweep exercises the search - // space, they exercise the ctypes bridge. They were registered in ctest but - // never invoked from here. All three are unittest-based, take no --gfx, and - // exit 0 on an internal skip, so they need no run_ok. + python3 ../dispatcher/tests/test_multi_abd_gpu_correctness.py && \ + run_ok python3 ../dispatcher/tests/test_rowcolquant_gpu_correctness.py --gfx ${arch} && \ + run_ok python3 ../dispatcher/tests/test_tensorquant_gpu_correctness.py --gfx ${arch}""" + // The grouped/multi_d/multi_abd tests are the bridge-level companions to the + // --variant sweeps below: the sweep exercises the search space, they exercise + // the ctypes bridge. They were registered in ctest but never invoked from + // here. All three are unittest-based, take no --gfx, and exit 0 on an + // internal skip, so they need no run_ok. + // + // rowcolquant and tensorquant sit here rather than in the gfx950 block + // because, unlike the other quant ops, both support gfx942 as well -- + // _SUPPORTED_ARCHES is ("gfx942", "gfx950") and their _fp8_uses_ocp mirrors + // the FNUZ/OCP split instead of assuming OCP. Their bridges landed in #10010. dispatcherGemmVariantsFor(arch).each { variant -> execute_cmd += " && \\\n " + dispatcherVariantCmd(arch, variant, tier) } @@ -1683,13 +1690,6 @@ def runDispatcherCorrectnessTests(String arch, String compiler, String tier) { // --variant sweep above and instead runs the two-config smoke test merged // in #10132. That test is unittest-based and self-gates to gfx950, so it // takes no --gfx and needs no run_ok (unlike its neighbours, which do). - // - // rowcolquant and tensorquant are absent but no longer blocked: #10010 - // landed their dispatcher bridges and ctest-registered GPU tests on develop - // after this lane's operator set was chosen. Wiring them in is a follow-up, - // not an oversight -- unlike their neighbours here, neither test self-gates - // on the detected arch (--gfx just defaults to gfx950), so adding them needs - // that skip path checked first or an unsupported node will run them red. if (arch == "gfx950") { execute_cmd += """ && \ run_ok python3 ../dispatcher/tests/test_bquant_gpu_correctness.py --gfx ${arch} && \ From df4cd47c3cc34076d3d3fa5b71936722ecd5e9ea Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Tue, 25 Aug 2026 19:42:32 -0400 Subject: [PATCH 25/51] ci(ck-tile): add gemm_streamk/aquant/bquant to dispatcher perf lane Extend RUN_DISPATCHER_PERF_TESTS to cover three new operators using existing tile_engine benchmark infrastructure: - gemm_streamk: added to the gfx942 lane alongside the original three ops (GEMM_STREAMK_DATATYPE/LAYOUT, benchmark_gemm_streamk_all, gemm_streamk_benchmark.py) - gemm_aquant, gemm_bquant: added in a new gfx950 parallel stage since both ops are fp8/bf8-only and require gfx950 native support (GEMM_AQUANT_DATATYPE/LAYOUT, GEMM_BQUANT_DATATYPE/LAYOUT, benchmark_gemm_aquant_all, benchmark_gemm_bquant_all) runDispatcherPerfTests() gains a gpuTarget parameter (default "gfx942") to drive the split. Both smoke and daily tiers get the new gfx950 stage. Co-Authored-By: Claude --- projects/composablekernel/Jenkinsfile | 44 +++++++++++-- .../composablekernel/groovy/vars/ck.groovy | 62 +++++++++++++------ 2 files changed, 84 insertions(+), 22 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index ef97e92f4fc6..98a9861fb841 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -138,11 +138,11 @@ pipeline { booleanParam( name: "RUN_DISPATCHER_PERF_TESTS", defaultValue: false, - description: "Smoke-tier benchmark of ahead-of-time tile_engine GEMM instances on gfx942: gemm_universal, batched_gemm, batched_contraction at fp16/rcr, one problem size, 500-instance budget (default: OFF)") + description: "Smoke-tier benchmark of ahead-of-time tile_engine GEMM instances on gfx942+gfx950: gemm_universal, batched_gemm, batched_contraction, gemm_streamk at fp16/rcr on gfx942; gemm_aquant, gemm_bquant at fp8/rcr on gfx950, one problem size, 500-instance budget (default: OFF)") booleanParam( name: "RUN_DISPATCHER_PERF_TESTS_DAILY", defaultValue: false, - description: "Full-tier benchmark of ahead-of-time tile_engine GEMM instances on gfx942: the same three ops across the fp8/fp16/bf8/bf16 x rcr/rrr/crr/ccr matrix, three problem sizes, 8000-instance budget (default: OFF)") + description: "Full-tier benchmark of ahead-of-time tile_engine GEMM instances on gfx942+gfx950: gemm_universal, batched_gemm, batched_contraction, gemm_streamk across fp8/fp16/bf8/bf16 x rcr/rrr/crr/ccr on gfx942; gemm_aquant, gemm_bquant across fp8/bf8 x rcr/rrr/ccr/crr on gfx950, three problem sizes, 8000-instance budget (default: OFF)") booleanParam( name: "RUN_DISPATCHER_CORRECTNESS_TESTS", defaultValue: false, @@ -644,7 +644,25 @@ pipeline { loadCk() ck.runOnHealthyNode(rocmnode("gfx942")) { deleteDir() - ck.runDispatcherPerfTests(params.BUILD_COMPILER, "smoke") + ck.runDispatcherPerfTests(params.BUILD_COMPILER, "smoke", "gfx942") + cleanWs() + } + } + } + } + stage("Run DISPATCHER_PERF Tests on gfx950") + { + when { + beforeAgent true + expression { params.RUN_DISPATCHER_PERF_TESTS.toBoolean() } + } + agent none + steps{ + script { + loadCk() + ck.runOnHealthyNode(rocmnode("gfx950")) { + deleteDir() + ck.runDispatcherPerfTests(params.BUILD_COMPILER, "smoke", "gfx950") cleanWs() } } @@ -672,7 +690,25 @@ pipeline { loadCk() ck.runOnHealthyNode(rocmnode("gfx942")) { deleteDir() - ck.runDispatcherPerfTests(params.BUILD_COMPILER, "daily") + ck.runDispatcherPerfTests(params.BUILD_COMPILER, "daily", "gfx942") + cleanWs() + } + } + } + } + stage("Run DISPATCHER_PERF_DAILY Tests on gfx950") + { + when { + beforeAgent true + expression { params.RUN_DISPATCHER_PERF_TESTS_DAILY.toBoolean() } + } + agent none + steps{ + script { + loadCk() + ck.runOnHealthyNode(rocmnode("gfx950")) { + deleteDir() + ck.runDispatcherPerfTests(params.BUILD_COMPILER, "daily", "gfx950") cleanWs() } } diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index c4684658798f..d95e5530fd23 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1515,7 +1515,7 @@ def runTileEngineGemmTests(String arch, String compiler) { // The tier is woven into every result filename. Both lanes write into // projects/composablekernel/build, so without it a build with both enabled // would have the second stage's archiveArtifacts overwrite the first's. -def runDispatcherPerfTests(String compiler, String tier) { +def runDispatcherPerfTests(String compiler, String tier, String gpuTarget = "gfx942") { def daily = (tier == "daily") def samplingTier = daily ? "daily" : "500" def gemmDatatype = daily ? "fp8;fp16;bf8;bf16" : "fp16" @@ -1525,23 +1525,49 @@ def runDispatcherPerfTests(String compiler, String tier) { : '"1024,1024,1024"' def problemConfigs = daily ? '"g=2;m=1024;n=1024;k=1024" "g=4;m=2048;n=2048;k=2048"' : '"g=2;m=1024;n=1024;k=1024"' - def execute_cmd = """ - cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ - -D BUILD_CK_TILE_ENGINE="ON" \ - -D CMAKE_CXX_COMPILER="${compiler}" \ - -D CMAKE_BUILD_TYPE=Release \ - -D GPU_TARGETS="gfx942" \ - -D GEMM_UNIVERSAL_DATATYPE="${gemmDatatype}" \ - -D GEMM_UNIVERSAL_LAYOUT="${gemmLayout}" \ - -D BATCHED_GEMM_DATATYPE="fp16" \ - -D BATCHED_GEMM_LAYOUT="rcr" \ - -D BATCHED_CONTRACTION_DATATYPE="fp16" \ - -D BATCHED_CONTRACTION_LAYOUT="rcr" \ - -D TILE_ENGINE_SAMPLING_TIER=${samplingTier} .. && \ - ninja -j${nthreads()} benchmark_gemm_universal_all benchmark_batched_gemm_all benchmark_batched_contraction_all && \ - python3 ../tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_universal_results_${tier}.json && \ - python3 ../tile_engine/ops/gemm/batched_gemm/batched_gemm_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json batched_gemm_results_${tier}.json && \ - python3 ../tile_engine/ops/gemm/batched_contraction/batched_contraction_benchmark.py . --problem-configs ${problemConfigs} --warmup 5 --repeat 5 --verbose --json batched_contraction_results_${tier}.json""" + // gemm_aquant and gemm_bquant are fp8/bf8-only; run on gfx950 where those + // data types are natively supported. gemm_streamk and the original three ops + // run on gfx942. + def quantDatatype = daily ? "fp8;bf8" : "fp8" + def quantLayout = daily ? "rcr;rrr;ccr;crr" : "rcr" + def execute_cmd + if (gpuTarget == "gfx950") { + execute_cmd = """ + cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ + -D BUILD_CK_TILE_ENGINE="ON" \ + -D CMAKE_CXX_COMPILER="${compiler}" \ + -D CMAKE_BUILD_TYPE=Release \ + -D GPU_TARGETS="gfx950" \ + -D GEMM_AQUANT_DATATYPE="${quantDatatype}" \ + -D GEMM_AQUANT_LAYOUT="${quantLayout}" \ + -D GEMM_BQUANT_DATATYPE="${quantDatatype}" \ + -D GEMM_BQUANT_LAYOUT="${quantLayout}" \ + -D TILE_ENGINE_SAMPLING_TIER=${samplingTier} .. && \ + ninja -j${nthreads()} benchmark_gemm_aquant_all benchmark_gemm_bquant_all && \ + python3 ../tile_engine/ops/gemm/block_scale_gemm/gemm_aquant/gemm_aquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_aquant_results_${tier}.json && \ + python3 ../tile_engine/ops/gemm/block_scale_gemm/gemm_bquant/gemm_bquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_bquant_results_${tier}.json""" + } else { + execute_cmd = """ + cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ + -D BUILD_CK_TILE_ENGINE="ON" \ + -D CMAKE_CXX_COMPILER="${compiler}" \ + -D CMAKE_BUILD_TYPE=Release \ + -D GPU_TARGETS="gfx942" \ + -D GEMM_UNIVERSAL_DATATYPE="${gemmDatatype}" \ + -D GEMM_UNIVERSAL_LAYOUT="${gemmLayout}" \ + -D BATCHED_GEMM_DATATYPE="fp16" \ + -D BATCHED_GEMM_LAYOUT="rcr" \ + -D BATCHED_CONTRACTION_DATATYPE="fp16" \ + -D BATCHED_CONTRACTION_LAYOUT="rcr" \ + -D GEMM_STREAMK_DATATYPE="${gemmDatatype}" \ + -D GEMM_STREAMK_LAYOUT="${gemmLayout}" \ + -D TILE_ENGINE_SAMPLING_TIER=${samplingTier} .. && \ + ninja -j${nthreads()} benchmark_gemm_universal_all benchmark_batched_gemm_all benchmark_batched_contraction_all benchmark_gemm_streamk_all && \ + python3 ../tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_universal_results_${tier}.json && \ + python3 ../tile_engine/ops/gemm/batched_gemm/batched_gemm_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json batched_gemm_results_${tier}.json && \ + python3 ../tile_engine/ops/gemm/batched_contraction/batched_contraction_benchmark.py . --problem-configs ${problemConfigs} --warmup 5 --repeat 5 --verbose --json batched_contraction_results_${tier}.json && \ + python3 ../tile_engine/ops/gemm_streamk/gemm_streamk_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_streamk_results_${tier}.json""" + } try { buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) } finally { From 07d1ab49b42e51ab595084ee60e701637399e027 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Tue, 25 Aug 2026 20:06:34 -0400 Subject: [PATCH 26/51] ci(ck-tile): drop the daily dispatcher lanes as out of scope The daily tiers were a second cost point for the same operator set, and that scope belongs in a follow-up rather than in the lane that first brings the dispatcher under CI. Removing them takes out both Jenkinsfile stages, their two parameters, and their entries in the 22:00 cron. With one tier left, the tier argument on runDispatcherPerfTests, runDispatcherCorrectnessTests, and dispatcherVariantCmd carried no information, so the daily ternaries collapse to their smoke values: a 500-instance AOT budget at fp16/rcr over one problem size for the perf lane, --budget 64 for the correctness sweep and 16 for each variant. The _ suffix on the result filenames existed only so a build with both lanes enabled would not have one stage's archiveArtifacts overwrite the other's. That cannot happen now, and the bare names match what the surrounding tile_engine stages already write. Co-Authored-By: Claude Opus 5 --- projects/composablekernel/Jenkinsfile | 121 ++---------------- .../composablekernel/groovy/vars/ck.groovy | 93 ++++++-------- 2 files changed, 46 insertions(+), 168 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index 98a9861fb841..0827c15d96f4 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -37,7 +37,7 @@ def loadCk() { //launch develop branch daily jobs CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;RUN_CK_TILE_FMHA_TESTS=true;RUN_PERFORMANCE_TESTS=true;FORCE_CI=true - 0 22 * * * % RUN_FULL_QA=true;DISABLE_DL_KERNELS=true;RUN_TILE_ENGINE_BASIC_TESTS=true;RUN_TILE_ENGINE_GEMM_TESTS=true;RUN_DISPATCHER_PERF_TESTS_DAILY=true;RUN_DISPATCHER_CORRECTNESS_TESTS_DAILY=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true + 0 22 * * * % RUN_FULL_QA=true;DISABLE_DL_KERNELS=true;RUN_TILE_ENGINE_BASIC_TESTS=true;RUN_TILE_ENGINE_GEMM_TESTS=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true 0 21 * * * % RUN_GROUPED_CONV_LARGE_CASES_TESTS=true;hipTensor_test=true;BUILD_GFX101=false;BUILD_GFX908=false;BUILD_GFX942=true;BUILD_GFX950=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true;BUILD_PACKAGES=true 0 19 * * * % BUILD_DOCKER=true;COMPILER_VERSION=develop;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true 0 17 * * * % BUILD_DOCKER=true;COMPILER_VERSION=therock;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true @@ -139,18 +139,10 @@ pipeline { name: "RUN_DISPATCHER_PERF_TESTS", defaultValue: false, description: "Smoke-tier benchmark of ahead-of-time tile_engine GEMM instances on gfx942+gfx950: gemm_universal, batched_gemm, batched_contraction, gemm_streamk at fp16/rcr on gfx942; gemm_aquant, gemm_bquant at fp8/rcr on gfx950, one problem size, 500-instance budget (default: OFF)") - booleanParam( - name: "RUN_DISPATCHER_PERF_TESTS_DAILY", - defaultValue: false, - description: "Full-tier benchmark of ahead-of-time tile_engine GEMM instances on gfx942+gfx950: gemm_universal, batched_gemm, batched_contraction, gemm_streamk across fp8/fp16/bf8/bf16 x rcr/rrr/crr/ccr on gfx942; gemm_aquant, gemm_bquant across fp8/bf8 x rcr/rrr/ccr/crr on gfx950, three problem sizes, 8000-instance budget (default: OFF)") booleanParam( name: "RUN_DISPATCHER_CORRECTNESS_TESTS", defaultValue: false, description: "Smoke-tier verification of JIT-compiled dispatcher kernels on gfx942 + gfx950: gemm_universal sweep at budget 64 plus parity, grouped, multi_d, multi_abd, batched_gemm, batched_contraction, rowcolquant, tensorquant on both arches, stream_k on gfx942, and mx_gemm + aquant + bquant + abquant on gfx950 (per-arch subset; default: OFF)") - booleanParam( - name: "RUN_DISPATCHER_CORRECTNESS_TESTS_DAILY", - defaultValue: false, - description: "Full-tier verification of JIT-compiled dispatcher kernels on gfx942 + gfx950: the same operator set as the smoke tier, at budget 500 for the gemm_universal sweep and 64 for each variant sweep (default: OFF)") booleanParam( name: "BUILD_INSTANCES_ONLY", defaultValue: false, @@ -644,7 +636,7 @@ pipeline { loadCk() ck.runOnHealthyNode(rocmnode("gfx942")) { deleteDir() - ck.runDispatcherPerfTests(params.BUILD_COMPILER, "smoke", "gfx942") + ck.runDispatcherPerfTests(params.BUILD_COMPILER, "gfx942") cleanWs() } } @@ -662,53 +654,7 @@ pipeline { loadCk() ck.runOnHealthyNode(rocmnode("gfx950")) { deleteDir() - ck.runDispatcherPerfTests(params.BUILD_COMPILER, "smoke", "gfx950") - cleanWs() - } - } - } - } - } - } - stage("Run DISPATCHER_PERF_DAILY Tests") - { - when { - beforeAgent true - expression { env.SHOULD_RUN_CI.toBoolean() } - } - parallel - { - stage("Run DISPATCHER_PERF_DAILY Tests on gfx942") - { - when { - beforeAgent true - expression { params.RUN_DISPATCHER_PERF_TESTS_DAILY.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx942")) { - deleteDir() - ck.runDispatcherPerfTests(params.BUILD_COMPILER, "daily", "gfx942") - cleanWs() - } - } - } - } - stage("Run DISPATCHER_PERF_DAILY Tests on gfx950") - { - when { - beforeAgent true - expression { params.RUN_DISPATCHER_PERF_TESTS_DAILY.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx950")) { - deleteDir() - ck.runDispatcherPerfTests(params.BUILD_COMPILER, "daily", "gfx950") + ck.runDispatcherPerfTests(params.BUILD_COMPILER, "gfx950") cleanWs() } } @@ -780,11 +726,11 @@ pipeline { } } } - // Both tiers fan out to gfx942 and gfx950. The quant operators (mx_gemm, + // The lane fans out to gfx942 and gfx950. The quant operators (mx_gemm, // aquant, bquant, abquant) are gfx950-only, so covering the full operator - // surface is impossible from a single node type. What separates smoke from - // daily is the search budget, not the operator count -- see the tier table - // on runDispatcherCorrectnessTests in groovy/vars/ck.groovy. + // surface is impossible from a single node type. Cost is held down by the + // search budget, not by dropping operators -- see the budget note on + // runDispatcherCorrectnessTests in groovy/vars/ck.groovy. stage("Run DISPATCHER_CORRECTNESS Tests") { when { @@ -805,7 +751,7 @@ pipeline { loadCk() ck.runOnHealthyNode(rocmnode("gfx942")) { deleteDir() - ck.runDispatcherCorrectnessTests("gfx942", params.BUILD_COMPILER, "smoke") + ck.runDispatcherCorrectnessTests("gfx942", params.BUILD_COMPILER) cleanWs() } } @@ -823,56 +769,7 @@ pipeline { loadCk() ck.runOnHealthyNode(rocmnode("gfx950")) { deleteDir() - ck.runDispatcherCorrectnessTests("gfx950", params.BUILD_COMPILER, "smoke") - cleanWs() - } - } - } - } - // No gfx1201 dispatcher stage: the dispatcher has no RDNA4 tile - // coverage, so such a stage could only ever run red. See the - // arch note on dispatcherGemmVariantsFor in groovy/vars/ck.groovy. - } - } - stage("Run DISPATCHER_CORRECTNESS_DAILY Tests") - { - when { - beforeAgent true - expression { env.SHOULD_RUN_CI.toBoolean() } - } - parallel - { - stage("Run DISPATCHER_CORRECTNESS_DAILY Tests on gfx942") - { - when { - beforeAgent true - expression { params.RUN_DISPATCHER_CORRECTNESS_TESTS_DAILY.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx942")) { - deleteDir() - ck.runDispatcherCorrectnessTests("gfx942", params.BUILD_COMPILER, "daily") - cleanWs() - } - } - } - } - stage("Run DISPATCHER_CORRECTNESS_DAILY Tests on gfx950") - { - when { - beforeAgent true - expression { params.RUN_DISPATCHER_CORRECTNESS_TESTS_DAILY.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx950")) { - deleteDir() - ck.runDispatcherCorrectnessTests("gfx950", params.BUILD_COMPILER, "daily") + ck.runDispatcherCorrectnessTests("gfx950", params.BUILD_COMPILER) cleanWs() } } diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index d95e5530fd23..9e0374ad11f2 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1503,33 +1503,21 @@ def runTileEngineGemmTests(String arch, String compiler) { buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) } -// Benchmark ahead-of-time tile_engine instances. Two coverage tiers: -// -// "smoke" -- one dtype, one layout, one problem size, and a 500-instance AOT -// budget. Sized to catch codegen/build/launch breakage on demand -// without booking a GPU node for the length of a full sweep. -// "daily" -- the full 4x4 dtype x layout matrix across three problem sizes at -// TILE_ENGINE_SAMPLING_TIER=daily, which maps to an 8000-instance -// budget in tile_engine/ops/gemm/CMakeLists.txt. -// -// The tier is woven into every result filename. Both lanes write into -// projects/composablekernel/build, so without it a build with both enabled -// would have the second stage's archiveArtifacts overwrite the first's. -def runDispatcherPerfTests(String compiler, String tier, String gpuTarget = "gfx942") { - def daily = (tier == "daily") - def samplingTier = daily ? "daily" : "500" - def gemmDatatype = daily ? "fp8;fp16;bf8;bf16" : "fp16" - def gemmLayout = daily ? "rcr;rrr;crr;ccr" : "rcr" - // nargs="+" on both drivers, so extra sizes are extra positional args. - def problemSizes = daily ? '"1024,1024,1024" "2048,2048,2048" "4096,4096,4096"' - : '"1024,1024,1024"' - def problemConfigs = daily ? '"g=2;m=1024;n=1024;k=1024" "g=4;m=2048;n=2048;k=2048"' - : '"g=2;m=1024;n=1024;k=1024"' +// Benchmark ahead-of-time tile_engine instances at smoke coverage: one dtype, +// one layout, one problem size, and a 500-instance AOT budget. Sized to catch +// codegen/build/launch breakage on demand without booking a GPU node for the +// length of a full sweep. +def runDispatcherPerfTests(String compiler, String gpuTarget = "gfx942") { + def samplingTier = "500" + def gemmDatatype = "fp16" + def gemmLayout = "rcr" + def problemSizes = '"1024,1024,1024"' + def problemConfigs = '"g=2;m=1024;n=1024;k=1024"' // gemm_aquant and gemm_bquant are fp8/bf8-only; run on gfx950 where those // data types are natively supported. gemm_streamk and the original three ops // run on gfx942. - def quantDatatype = daily ? "fp8;bf8" : "fp8" - def quantLayout = daily ? "rcr;rrr;ccr;crr" : "rcr" + def quantDatatype = "fp8" + def quantLayout = "rcr" def execute_cmd if (gpuTarget == "gfx950") { execute_cmd = """ @@ -1544,8 +1532,8 @@ def runDispatcherPerfTests(String compiler, String tier, String gpuTarget = "gfx -D GEMM_BQUANT_LAYOUT="${quantLayout}" \ -D TILE_ENGINE_SAMPLING_TIER=${samplingTier} .. && \ ninja -j${nthreads()} benchmark_gemm_aquant_all benchmark_gemm_bquant_all && \ - python3 ../tile_engine/ops/gemm/block_scale_gemm/gemm_aquant/gemm_aquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_aquant_results_${tier}.json && \ - python3 ../tile_engine/ops/gemm/block_scale_gemm/gemm_bquant/gemm_bquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_bquant_results_${tier}.json""" + python3 ../tile_engine/ops/gemm/block_scale_gemm/gemm_aquant/gemm_aquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_aquant_results.json && \ + python3 ../tile_engine/ops/gemm/block_scale_gemm/gemm_bquant/gemm_bquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_bquant_results.json""" } else { execute_cmd = """ cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ @@ -1563,10 +1551,10 @@ def runDispatcherPerfTests(String compiler, String tier, String gpuTarget = "gfx -D GEMM_STREAMK_LAYOUT="${gemmLayout}" \ -D TILE_ENGINE_SAMPLING_TIER=${samplingTier} .. && \ ninja -j${nthreads()} benchmark_gemm_universal_all benchmark_batched_gemm_all benchmark_batched_contraction_all benchmark_gemm_streamk_all && \ - python3 ../tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_universal_results_${tier}.json && \ - python3 ../tile_engine/ops/gemm/batched_gemm/batched_gemm_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json batched_gemm_results_${tier}.json && \ - python3 ../tile_engine/ops/gemm/batched_contraction/batched_contraction_benchmark.py . --problem-configs ${problemConfigs} --warmup 5 --repeat 5 --verbose --json batched_contraction_results_${tier}.json && \ - python3 ../tile_engine/ops/gemm_streamk/gemm_streamk_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_streamk_results_${tier}.json""" + python3 ../tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_universal_results.json && \ + python3 ../tile_engine/ops/gemm/batched_gemm/batched_gemm_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json batched_gemm_results.json && \ + python3 ../tile_engine/ops/gemm/batched_contraction/batched_contraction_benchmark.py . --problem-configs ${problemConfigs} --warmup 5 --repeat 5 --verbose --json batched_contraction_results.json && \ + python3 ../tile_engine/ops/gemm_streamk/gemm_streamk_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_streamk_results.json""" } try { buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) @@ -1576,7 +1564,7 @@ def runDispatcherPerfTests(String compiler, String tier, String gpuTarget = "gfx // reasoning as runDispatcherCorrectnessTests: execute_cmd runs from // projects/composablekernel/build while archiveArtifacts resolves against // the workspace root, so the build-relative path is spelled out in full. - archiveArtifacts artifacts: "projects/composablekernel/build/*_results_${tier}.json", + archiveArtifacts artifacts: "projects/composablekernel/build/*_results.json", allowEmptyArchive: true } } @@ -1610,13 +1598,12 @@ def dispatcherGemmVariantsFor(String arch) { // would enumerate configs that variant cannot build). multi_d likewise takes the // runner's default --elementwise-op PassThrough. // -// Both tiers run every variant, so the budget is what separates them. It stays -// far under the standard sweep's (500 daily / 64 smoke) because each variant +// The budget stays far under the standard sweep's 64 because each variant // JIT-compiles its own .so set and these run in addition to, not instead of, -// that sweep -- 16 is enough for smoke to prove each variant still builds and -// matches its reference, without paying the daily lane's coverage. -def dispatcherVariantCmd(String arch, String variant, String tier) { - def budget = (tier == "daily") ? 64 : 16 +// that sweep -- 16 is enough to prove each variant still builds and matches its +// reference. +def dispatcherVariantCmd(String arch, String variant) { + def budget = 16 return """python3 ../dispatcher/tests/test_gemm_search_space.py \ --variant ${variant} \ --arch ${arch} \ @@ -1624,28 +1611,22 @@ def dispatcherVariantCmd(String arch, String variant, String tier) { --warmup 5 \ --repeat 5 \ --size 1024 \ - --json dispatcher_${variant}_results_${tier}.json""" + --json dispatcher_${variant}_results.json""" } // Verify JIT-compiled dispatcher kernels against a host reference. // -// BOTH tiers run the same operator set for the arch -- every dispatcher operator +// The lane runs the whole operator set for the arch -- every dispatcher operator // with a bridge on develop -- 13 of them. A developer touching the bridge needs // pre-merge signal on all of them, and a lane covering four cannot give that. -// The tier controls cost only: -// -// "smoke" -- standard sweep at --budget 64, variants at --budget 16. -// "daily" -- standard sweep at --budget 500, variants at --budget 64. -// -// The dtype and layout lists stay at the full 4x4 in both tiers on purpose: the -// sweep samples per stratum, so even 64 spreads ~4 configs across all 16 dtype x -// layout combinations rather than testing one combination deeply. +// Cost is held down by the search budget instead: --budget 64 for the standard +// sweep, --budget 16 for each variant sweep. // -// As in runDispatcherPerfTests, result filenames carry the tier so a build with -// both lanes enabled does not have one stage's artifacts clobber the other's. -def runDispatcherCorrectnessTests(String arch, String compiler, String tier) { - def daily = (tier == "daily") - def budget = daily ? 500 : 64 +// The dtype and layout lists stay at the full 4x4 on purpose: the sweep samples +// per stratum, so even 64 spreads ~4 configs across all 16 dtype x layout +// combinations rather than testing one combination deeply. +def runDispatcherCorrectnessTests(String arch, String compiler) { + def budget = 64 // run_ok wraps the script-style tests, which signal a clean skip with exit // 77. execute_cmd is one &&-joined string, so without this an unsupported // arch or a missing ml_dtypes would abort the lane at that point and every @@ -1670,7 +1651,7 @@ def runDispatcherCorrectnessTests(String arch, String compiler, String tier) { --warmup 5 \ --repeat 5 \ --size 1024 \ - --json dispatcher_gemm_results_${tier}.json && \ + --json dispatcher_gemm_results.json && \ python3 ../dispatcher/tests/test_gemm_parity.py && \ run_ok python3 ../dispatcher/tests/test_batched_gemm_gpu_correctness.py --gfx ${arch} && \ run_ok python3 ../dispatcher/tests/test_batched_contraction_gpu_correctness.py --gfx ${arch} && \ @@ -1690,7 +1671,7 @@ def runDispatcherCorrectnessTests(String arch, String compiler, String tier) { // _SUPPORTED_ARCHES is ("gfx942", "gfx950") and their _fp8_uses_ocp mirrors // the FNUZ/OCP split instead of assuming OCP. Their bridges landed in #10010. dispatcherGemmVariantsFor(arch).each { variant -> - execute_cmd += " && \\\n " + dispatcherVariantCmd(arch, variant, tier) + execute_cmd += " && \\\n " + dispatcherVariantCmd(arch, variant) } // Stream-K has its own registry-level driver test; its defaults are the // full 4x4 matrix (48 driver compiles / 96 GPU runs), so CI pins one @@ -1708,7 +1689,7 @@ def runDispatcherCorrectnessTests(String arch, String compiler, String tier) { } // The block-scale quant operators are gfx950-only: their utils modules pin // _DEFAULT_GFX_ARCH = "gfx950" and the tests self-gate to it. This is why the - // smoke tier fans out to a gfx950 node as well as gfx942 -- see the stage + // lane fans out to a gfx950 node as well as gfx942 -- see the stage // comment in the Jenkinsfile. // // mx_gemm has no expand_sweep -- mx_gemm_utils exposes only @@ -1734,7 +1715,7 @@ def runDispatcherCorrectnessTests(String arch, String compiler, String tier) { // against the workspace root. A bare "dispatcher_*_results.json" matches // nothing there, and allowEmptyArchive hides that it matched nothing. // Glob, not one name: each variant writes its own dispatcher__*.json. - archiveArtifacts artifacts: "projects/composablekernel/build/dispatcher_*_results_${tier}.json", + archiveArtifacts artifacts: "projects/composablekernel/build/dispatcher_*_results.json", allowEmptyArchive: true } } From 87cf06327127e89772e39c21a83159e6b5427c19 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Wed, 26 Aug 2026 15:35:54 -0400 Subject: [PATCH 27/51] ci(ck-tile): stop the dispatcher lane reporting false greens Three fixes to the RUN_DISPATCHER_CORRECTNESS_TESTS lane, all of which make it report what it actually ran. test_gemm_parity.py returned 0 when it skipped. It is not ctest-registered, so the exit code is the only signal the lane gets: a CPU-only or hipcc-less runner reported a green PASS for the 60-case matrix -- the lane's only int8 coverage, since test_gemm_search_space has no int8 stratum -- without building a single kernel. It now returns 77, and the caller wraps it in run_ok. run_ok used `"$@"; rc=$?`, which is unsafe as the final element of an && list. buildAndTest emits `sh cmd` with no shebang, so this runs under /bin/sh -xe with set -e live, and set -e is suppressed inside a function body only while the call is a non-final element. Both arch chains end on a run_ok, so a legitimate skip there killed the shell before rc=$? could run. `"$@" || rc=$?` puts the call in a tested context and is immune regardless of position. The gfx950 fp8/bf8 sweep strata were nominal. The device side JIT-builds without -DCK_TILE_USE_OCP_FP8 (unlike the quant bridges, which pass it) so the kernel takes the FNUZ branch while gfx950's fp8 MFMA is OCP; the host codec in gemm_utils.py is FNUZ-only and documents that it is. They passed only on loose tolerances (fp8 1.5e-1, bf8 3.0e-1) -- validating an encoding production never ships. dispatcherSweepDtypesFor narrows gfx950 to fp16,bf16 until the codec is arch-aware and the JIT flags are passed; both are needed, neither works alone. Also pins ml_dtypes==0.6.0. It supplies both the OCP and FNUZ fp8 pairs and the tests choose between them per arch, so an unpinned upgrade would move reference values under CI on an image rebuild. Co-Authored-By: Claude Opus 5 --- projects/composablekernel/Dockerfile | 10 +- .../dispatcher/tests/test_gemm_parity.py | 12 +- .../composablekernel/groovy/vars/ck.groovy | 135 ++++++++++++++++-- 3 files changed, 144 insertions(+), 13 deletions(-) diff --git a/projects/composablekernel/Dockerfile b/projects/composablekernel/Dockerfile index 61c9f4ad1819..34c3871944ac 100644 --- a/projects/composablekernel/Dockerfile +++ b/projects/composablekernel/Dockerfile @@ -100,7 +100,15 @@ RUN set -x && \ wget https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \ dpkg -i dumb-init_*.deb && rm dumb-init_*.deb && \ # Install packages for processing the performance results - pip3 install --break-system-packages --upgrade pytest pymysql pandas==2.2.3 sqlalchemy==2.0.3 setuptools-rust setuptools sshtunnel==0.4.0 ml_dtypes && \ +# ml_dtypes is pinned like its neighbours, and for a sharper reason than most: +# it supplies both float8_e4m3fn/float8_e5m2 (OCP) and float8_e4m3fnuz/ +# float8_e5m2fnuz (FNUZ), and the dispatcher tests pick between those pairs per +# arch. An unpinned upgrade would silently move fp8 encoding semantics -- and +# the reference values every parity/correctness test compares against -- from +# under CI on an image rebuild. 0.6.0 is what this image already resolved on +# Ubuntu 24.04 / Python 3.12, so this records current behaviour rather than +# bumping it. + pip3 install --break-system-packages --upgrade pytest pymysql pandas==2.2.3 sqlalchemy==2.0.3 setuptools-rust setuptools sshtunnel==0.4.0 ml_dtypes==0.6.0 && \ # Add render group groupadd -f render && \ # Install the new rocm-cmake version diff --git a/projects/composablekernel/dispatcher/tests/test_gemm_parity.py b/projects/composablekernel/dispatcher/tests/test_gemm_parity.py index b9d1cd1cd9ab..0814e11ee650 100644 --- a/projects/composablekernel/dispatcher/tests/test_gemm_parity.py +++ b/projects/composablekernel/dispatcher/tests/test_gemm_parity.py @@ -102,6 +102,16 @@ _LAYOUT_WORD = {"r": "row", "c": "col"} +# Same convention as the sibling *_gpu_correctness.py tests and as +# SKIP_RETURN_CODE in dispatcher/tests/CMakeLists.txt. This file is not +# ctest-registered, so the exit code is the only signal a CI lane gets: returning +# 0 from _main() on a CPU-only or hipcc-less runner would report a green PASS for +# the 60-case matrix below -- the lane's only int8 coverage -- without a single +# kernel having been built. The caller in groovy/vars/ck.groovy wraps this in +# run_ok, which translates 77 into a logged skip and lets any other non-zero +# exit fail the lane. +SKIP_EXIT = 77 + def _emulate_input(x: np.ndarray, dtype: str) -> np.ndarray: """Round an fp32 operand to the kernel's storage dtype so the CPU reference @@ -259,7 +269,7 @@ def _main() -> int: reason = _gpu_environment_reason() if reason: print(f"SKIP: {reason}") - return 0 + return SKIP_EXIT arch = detect_gpu_arch() print("=" * 78) diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 9e0374ad11f2..536155945cef 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1513,11 +1513,19 @@ def runDispatcherPerfTests(String compiler, String gpuTarget = "gfx942") { def gemmLayout = "rcr" def problemSizes = '"1024,1024,1024"' def problemConfigs = '"g=2;m=1024;n=1024;k=1024"' - // gemm_aquant and gemm_bquant are fp8/bf8-only; run on gfx950 where those - // data types are natively supported. gemm_streamk and the original three ops - // run on gfx942. + // gemm_aquant and gemm_bquant run on gfx950 because their tests and utils + // modules self-gate to it -- SUPPORTED_ARCHS = ("gfx950",) and + // _DEFAULT_GFX_ARCH = "gfx950" -- not because gfx942 lacks fp8. gfx942 has + // fp8 (FNUZ) and is in DESIRED_TARGETS; widening these ops to it means + // relaxing those guards first. gemm_streamk and the original three ops run + // on gfx942. def quantDatatype = "fp8" def quantLayout = "rcr" + // Stream-K below is tile_engine/ops/gemm_streamk/ (top level), not + // tile_engine/ops/gemm/gemm_streamk/. Two paths exist and only the first is + // real: it holds the CMakeLists, the instance builder and the benchmark. The + // nested one contains a lone configs/default_config.json that nothing in the + // tree reads. Edit the top-level tree. def execute_cmd if (gpuTarget == "gfx950") { execute_cmd = """ @@ -1591,22 +1599,63 @@ def dispatcherGemmVariantsFor(String arch) { } } +// Which dtypes the GEMM sweeps may ask for on each arch. +// +// gfx950 is fp16/bf16 only, and the reason is not that the hardware lacks fp8 -- +// it is that the dispatcher's fp8 path is gfx942-shaped on both ends: +// +// * Device: CMakeLists.txt (see the CK_TILE_USE_OCP_FP8 block) compiles gfx950 +// and gfx12 with -DCK_TILE_USE_OCP_FP8. The quant bridges replicate that in +// their own JIT flags (grouped_gemm_{a,b,rowcol,tensor}quant_utils.py all +// append -DCK_USE_OCP_FP8 -DCK_TILE_USE_OCP_FP8 for gfx950), but the GEMM +// sweep path does not: gemm_utils.py and ctypes_utils.py hand hipcc only +// --offload-arch. So a gfx950 node here JIT-builds kernels that fall into +// the FNUZ #else branch of ck_tile/core/numeric/float8.hpp, while gfx950's +// fp8 MFMA and its __gfx950__ conversion intrinsics are OCP. That asymmetry +// is why the quant operators keep their fp8 coverage on gfx950 and the +// sweeps below lose theirs. +// * Host: the fp8/bf8 codec in gemm_utils.py is FNUZ-only and says so -- +// "gfx950/MI350 uses the OCP fp8 format instead; this codec targets the +// gfx942 default and the OCP path needs separate handling." +// +// Those strata therefore validate an encoding production never ships, and they +// pass only because the parity tolerances are loose (fp8 1.5e-1, bf8 3.0e-1). +// test_streamk_gpu_correctness.py already refuses gfx950 for exactly this +// reason. Re-widening this list is not the fix; the fix is to port +// test_rowcolquant_gpu_correctness.py's _fp8_uses_ocp() into gemm_utils AND to +// pass -DCK_TILE_USE_OCP_FP8 in the JIT flags. Both, or neither works. +def dispatcherSweepDtypesFor(String arch) { + switch (arch) { + case "gfx942": return "fp16,bf16,fp8,bf8" + case "gfx950": return "fp16,bf16" + default: return "fp16,bf16" + } +} + // One test_gemm_search_space.py invocation for a non-standard GEMM variant. // -// dtypes/layouts are deliberately omitted: the runner picks per-variant defaults +// layouts are deliberately omitted: the runner picks per-variant defaults // (multi_d/multi_abd are fp16/rcrr-only, so naming the standard 4x4 matrix here // would enumerate configs that variant cannot build). multi_d likewise takes the // runner's default --elementwise-op PassThrough. // +// dtypes are omitted for the same reason -- except for "grouped", whose +// _VARIANT_DEFAULTS entry in test_gemm_search_space.py is the full +// fp16,bf16,fp8,bf8 list and would otherwise put fp8/bf8 back on gfx950 through +// the side door. multi_d/multi_abd default to fp16 alone and must keep doing so, +// which is why this is not applied to every variant. stream_k shares the full +// list but is gfx942-only, so it is unaffected either way. +// // The budget stays far under the standard sweep's 64 because each variant // JIT-compiles its own .so set and these run in addition to, not instead of, // that sweep -- 16 is enough to prove each variant still builds and matches its // reference. def dispatcherVariantCmd(String arch, String variant) { def budget = 16 + def dtypeArg = (variant == "grouped") ? " --dtypes ${dispatcherSweepDtypesFor(arch)}" : "" return """python3 ../dispatcher/tests/test_gemm_search_space.py \ --variant ${variant} \ - --arch ${arch} \ + --arch ${arch}${dtypeArg} \ --budget ${budget} \ --warmup 5 \ --repeat 5 \ @@ -1622,9 +1671,60 @@ def dispatcherVariantCmd(String arch, String variant) { // Cost is held down by the search budget instead: --budget 64 for the standard // sweep, --budget 16 for each variant sweep. // -// The dtype and layout lists stay at the full 4x4 on purpose: the sweep samples -// per stratum, so even 64 spreads ~4 configs across all 16 dtype x layout -// combinations rather than testing one combination deeply. +// The layout list stays at the full 4 on purpose, as does the dtype list on +// gfx942: the sweep samples per stratum, so even 64 spreads ~4 configs across +// all 16 dtype x layout combinations rather than testing one combination +// deeply. gfx950 runs 2 dtypes x 4 layouts -- see dispatcherSweepDtypesFor. +// +// --------------------------------------------------------------------------- +// Coverage against the 13 dispatcher operators with a bridge on develop. +// +// All 13 are invoked. Six are covered across their full documented dtype x +// layout x arch surface: Multi-ABD, Batched GEMM, MX Gemm, grouped_gemm_bquant, +// grouped_gemm_rowcol, grouped_gemm_tensor. The other seven run, but over a +// narrowed surface -- listed under "gaps" below. +// +// One non-obvious detail: gemm_universal's int8 coverage comes from +// test_gemm_parity.py's _INT_DTYPES, not from the sweep. test_gemm_search_space +// has no int8 stratum, so dropping the parity call would drop int8 entirely. +// That is why it is invoked here and why its skip path now exits 77 rather +// than 0. +// +// Known gaps, deliberately not closed here. Each needs a code change outside +// this lane; none is a matter of adding a flag to the commands below, except +// where noted: +// +// StreamK on gfx950 test_streamk_gpu_correctness.py pins +// SUPPORTED_ARCHS = ("gfx942",); blocked on +// the arch-aware fp8 codec. +// gemm_universal + Grouped, Blocked on the same thing from the other +// gfx950 fp8/bf8 side: the sweep's JIT path never passes +// -DCK_TILE_USE_OCP_FP8, unlike the quant +// bridges. See dispatcherSweepDtypesFor. +// Multi-D layouts rrrr, ccrr, Flag-only, and deferred by choice, not by +// crrr blocker -- the bridge handles all four (see +// gemm_multi_d_full_benchmark.py). Only rcrr +// runs today. +// Batched Contraction bf16, Test is fp16-only and takes no dtype flag; +// fp32 batched_contraction_utils exposes only +// default_fp16_config, though its _np_dtype() +// already resolves bf16 and fp32. +// grouped_gemm_aquant/abquant Tests self-gate to gfx950 and their config +// on gfx942, and non-rcr builders hardcode rcr. +// layouts +// +// Seven further operators (preshuffle, the non-grouped gemm_*quant family, +// batched-contraction multi-ABD) have no dispatcher bridge on develop and so +// cannot be covered from here at all. preshuffle is a partial exception: it +// exists in gemm_utils but is absent from test_gemm_search_space's _VARIANTS, +// so wiring it up is a real change, not a flag. +// +// The first two gaps unblock together and neither is fixable alone: the fix is +// _fp8_uses_ocp() (test_rowcolquant_gpu_correctness.py) ported into gemm_utils +// for the host side, plus -DCK_TILE_USE_OCP_FP8 in the sweep's JIT compile +// flags for the device side -- copying what the quant bridges already do. +// Tracked as a follow-up. +// --------------------------------------------------------------------------- def runDispatcherCorrectnessTests(String arch, String compiler) { def budget = 64 // run_ok wraps the script-style tests, which signal a clean skip with exit @@ -1633,8 +1733,21 @@ def runDispatcherCorrectnessTests(String arch, String compiler) { // later operator would silently never run. Only 77 is swallowed; any other // non-zero exit still fails the lane. A function definition itself exits 0, // so it chains cleanly ahead of cmake. + // + // "$@" || rc=$?, not "$@"; rc=$? -- and the difference is not cosmetic. + // buildAndTest emits `sh cmd` with no shebang, so the Durable Task plugin + // runs this under /bin/sh -xe and set -e is live. set -e is suppressed + // inside a function body only while the call is a NON-FINAL element of an + // && list; when run_ok is the last element, set -e applies inside the body + // and a bare "$@" exiting 77 kills the shell before rc=$? can run. Both + // arch chains below end on a run_ok (gfx942 on test_streamk_gpu_correctness, + // gfx950 on test_abquant_gpu_correctness), so a legitimate skip there would + // fail the lane. `cmd || rc=$?` puts the call in a tested context, which is + // immune to set -e regardless of where run_ok is called from. Keep it that + // way. rc=0 is a plain assignment, not `local`: /bin/sh is not bash and + // local is not POSIX. def execute_cmd = """ - run_ok() { "\$@"; rc=\$?; if [ \$rc -eq 77 ]; then echo "SKIP(77): \$*"; return 0; fi; return \$rc; } && \ + run_ok() { rc=0; "\$@" || rc=\$?; if [ \$rc -eq 77 ]; then echo "SKIP(77): \$*"; return 0; fi; return \$rc; } && \ cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ -D CMAKE_CXX_COMPILER="${compiler}" \ -D CMAKE_BUILD_TYPE=Release \ @@ -1645,14 +1758,14 @@ def runDispatcherCorrectnessTests(String arch, String compiler) { ninja -j${nthreads()} ck_tile_dispatcher dispatcher_gemm_lib && \ python3 ../dispatcher/tests/test_gemm_search_space.py \ --arch ${arch} \ - --dtypes fp16,bf16,fp8,bf8 \ + --dtypes ${dispatcherSweepDtypesFor(arch)} \ --layouts rcr,rrr,crr,ccr \ --budget ${budget} \ --warmup 5 \ --repeat 5 \ --size 1024 \ --json dispatcher_gemm_results.json && \ - python3 ../dispatcher/tests/test_gemm_parity.py && \ + run_ok python3 ../dispatcher/tests/test_gemm_parity.py && \ run_ok python3 ../dispatcher/tests/test_batched_gemm_gpu_correctness.py --gfx ${arch} && \ run_ok python3 ../dispatcher/tests/test_batched_contraction_gpu_correctness.py --gfx ${arch} && \ python3 ../dispatcher/tests/test_grouped_gemm_gpu_correctness.py && \ From c164dd4ca68898fb8aed95b8bc9904c7e93f8607 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Wed, 26 Aug 2026 16:10:50 -0400 Subject: [PATCH 28/51] fix(ck-tile): make the dispatcher fp8/bf8 encoding follow the target arch On gfx950 the dispatcher had three fp8 encodings in play at once. The device pass of config.hpp saw __gfx950__ and picked OCP; the host pass of the same header fell back to FNUZ; and the numpy reference in gemm_utils was unconditionally FNUZ. Nothing crashed -- OCP and FNUZ differ by one in the exponent bias, so every value was simply off by a factor of two, and the gfx950 fp8/bf8 strata passed only because the parity gates are loose enough to swallow it. That is why the sweep list was narrowed to fp16/bf16 and why test_streamk_gpu_correctness pins gfx942. The device and host halves have to move together, so both are fixed here: * dispatcher_common gains fp8_uses_ocp() and ocp_arch_defines() as the single predicate. The two byte-identical _fp8_uses_ocp() copies in the rowcolquant and tensorquant tests now import it, the four quant bridges that had their own verbatim arch_defines block share it, and abquant -- which had no arch defines at all and worked only because both ends happened to guess FNUZ -- gets them for the first time. * gemm_utils._build_compile_jobs passes those defines to hipcc, so the device pass stops guessing from the arch macro. * The host codec is arch-aware. _fp8_decode_table carries both formats, including the parts that are not just a bias shift: OCP e4m3fn has no Inf and reserves only the all-1s mantissa for NaN, OCP e5m2 is IEEE-like with real infinities, and FNUZ has neither -- its 0x80 slot is the sole NaN rather than -0. numpy_dtype_for picks e4m3fn/e5m2 vs e4m3fnuz/e5m2fnuz the same way. Encoding saturates to the max finite value in all four formats, comparing in float64 because at the top of the range the gap between the two largest magnitudes is below fp32 resolution and would otherwise tie. The codec is checked against ml_dtypes as an independent oracle: all four decode tables match bit-for-bit across all 256 bytes, every representable value round-trips exactly, and saturation lands on the max finite value rather than Inf. With that in place gfx950 gets its fp8/bf8 sweep coverage back. The parity gates are left as they are on purpose. They were wide enough to hide this on gfx950, but gfx942 was FNUZ-correct all along and still needs the room for 3-/2-bit quantization, so tightening them is a measurement to make on a GPU rather than something to deduce from this change. --- .../dispatcher/python/dispatcher_common.py | 57 ++++++ .../dispatcher/python/fmha_utils.py | 7 + .../dispatcher/python/gemm_utils.py | 193 +++++++++++++----- .../python/grouped_gemm_abquant_utils.py | 12 ++ .../python/grouped_gemm_aquant_utils.py | 14 +- .../python/grouped_gemm_bquant_utils.py | 11 +- .../python/grouped_gemm_rowcolquant_utils.py | 11 +- .../python/grouped_gemm_tensorquant_utils.py | 11 +- .../dispatcher/tests/test_gemm_parity.py | 12 +- .../dispatcher/tests/test_gemm_utils.py | 120 ++++++++++- .../tests/test_rowcolquant_gpu_correctness.py | 13 +- .../tests/test_tensorquant_gpu_correctness.py | 13 +- .../composablekernel/groovy/vars/ck.groovy | 38 ++-- 13 files changed, 393 insertions(+), 119 deletions(-) diff --git a/projects/composablekernel/dispatcher/python/dispatcher_common.py b/projects/composablekernel/dispatcher/python/dispatcher_common.py index 5254df6d4684..8945cb05cb1b 100644 --- a/projects/composablekernel/dispatcher/python/dispatcher_common.py +++ b/projects/composablekernel/dispatcher/python/dispatcher_common.py @@ -101,6 +101,63 @@ def detect_gpu_arch(fallback: str = "gfx942") -> str: return fallback +# ============================================================================ +# fp8 / bf8 encoding format per architecture +# ============================================================================ +# +# CK has two incompatible 8-bit float encodings and picks between them per arch: +# +# OCP (gfx950, gfx12): e4m3fn / e5m2 -- exponent bias 7 / 15 +# FNUZ (everything else, notably gfx942): e4m3fnuz / e5m2fnuz -- bias 8 / 16 +# +# include/ck_tile/core/config.hpp:361-371 resolves this at compile time, but only +# the *device* pass sees __gfx950__; the host pass of the very same header falls +# back to FNUZ. So a host-side reference encoder that guesses from the header ends +# up disagreeing with the kernel it is validating. Every caller must therefore +# decide from the target arch string, and pass -DCK_TILE_USE_OCP_FP8 explicitly so +# both compiler passes agree. These helpers are the single source of truth for +# both halves of that contract. + + +def fp8_uses_ocp(arch: Optional[str]) -> bool: + """True iff `arch` uses the OCP fp8/bf8 encoding rather than FNUZ. + + Mirrors the __gfx950__ / __gfx12__ test in include/ck_tile/core/config.hpp. + Use this to select the host-side codec (ml_dtypes.float8_e4m3fn vs + float8_e4m3fnuz) so it matches the bytes the kernel actually produces. + """ + a = (arch or "").lower() + return a.startswith("gfx950") or a.startswith("gfx12") + + +def ocp_arch_defines(arch: Optional[str]) -> List[str]: + """hipcc defines that pin the fp8/bf8 encoding for `arch`. + + Returned for OCP archs only; FNUZ is the default for both compiler passes and + needs no define. Passing these makes the host pass agree with the device pass + instead of silently falling back to FNUZ. + """ + if not fp8_uses_ocp(arch): + return [] + return ["-DCK_USE_OCP_FP8", "-DCK_TILE_USE_OCP_FP8"] + + +def arch_feature_defines(arch: Optional[str]) -> List[str]: + """`ocp_arch_defines` plus the gfx950 MX / scale-MFMA enablement defines. + + CMake sets CK_USE_NATIVE_MX_SUPPORT and CK_GFX950_SUPPORT for gfx950 builds; + they are absent in the standalone hipcc JIT path, so bridges that compile + block-scaled kernels must add them here. Note CK_GFX950_SUPPORT also selects + the warp-tile K in tile_gemm_shape.hpp get_k_warp_tile(), so a config whose + warp_tile_k was tuned for gfx950 will silently produce zeros if built for + gfx942 -- keep warp_tile_k arch-aware alongside these defines. + """ + defines = ocp_arch_defines(arch) + if (arch or "").lower().startswith("gfx950"): + defines = defines + ["-DCK_USE_NATIVE_MX_SUPPORT", "-DCK_GFX950_SUPPORT"] + return defines + + # ============================================================================ # Architecture Filter Data # ============================================================================ diff --git a/projects/composablekernel/dispatcher/python/fmha_utils.py b/projects/composablekernel/dispatcher/python/fmha_utils.py index 5d3d0854960b..dcd67612f63c 100644 --- a/projects/composablekernel/dispatcher/python/fmha_utils.py +++ b/projects/composablekernel/dispatcher/python/fmha_utils.py @@ -1236,6 +1236,13 @@ def fmha_compile_flags(arch: str, hipcc: str = "", family: str = "") -> List[str ] if arch.startswith("gfx9"): flags.append("-DCK_TILE_FMHA_FWD_FAST_EXP2=1") + # NOTE: OCP fp8 is forced on every gfx9 arch, including gfx942 whose CK + # default is FNUZ (see dispatcher_common.fp8_uses_ocp). That is not the + # host/device split-brain the GEMM bridges had -- the define lands on both + # compiler passes, so they agree -- but it does mean fmha fp8 on gfx942 + # runs a software OCP codec rather than the arch-native FNUZ one. + # Left as-is deliberately: flipping it is a numeric change to an operator + # outside this lane's coverage matrix and needs its own verification pass. flags.append("-DCK_TILE_USE_OCP_FP8") flags.append("-DCK_GFX950_SUPPORT") flags.append("-DCK_USE_GFX950") diff --git a/projects/composablekernel/dispatcher/python/gemm_utils.py b/projects/composablekernel/dispatcher/python/gemm_utils.py index f09dfe51f6ad..4e8f15f304a1 100644 --- a/projects/composablekernel/dispatcher/python/gemm_utils.py +++ b/projects/composablekernel/dispatcher/python/gemm_utils.py @@ -66,12 +66,24 @@ try: - # Reuse the single canonical amd-smi bridge instead of re-implementing it. + # Reuse the single canonical amd-smi bridge instead of re-implementing it, + # and the single source of truth for the fp8/bf8 encoding format per arch. from dispatcher_common import _detect_gpu_arch_via_amd_smi + from dispatcher_common import fp8_uses_ocp as _fp8_uses_ocp + from dispatcher_common import ocp_arch_defines as _ocp_arch_defines except Exception: # noqa: BLE001 - standalone use without dispatcher_common on path def _detect_gpu_arch_via_amd_smi() -> Optional[str]: return None + def _fp8_uses_ocp(arch: Optional[str]) -> bool: + a = (arch or "").lower() + return a.startswith("gfx950") or a.startswith("gfx12") + + def _ocp_arch_defines(arch: Optional[str]) -> List[str]: + if not _fp8_uses_ocp(arch): + return [] + return ["-DCK_USE_OCP_FP8", "-DCK_TILE_USE_OCP_FP8"] + @functools.lru_cache(maxsize=1) def _get_arch() -> str: @@ -179,11 +191,13 @@ def _cap(flag: bool) -> str: } -def numpy_dtype_for(dtype: str): +def numpy_dtype_for(dtype: str, use_ocp: Optional[bool] = None): """Return the numpy dtype object used for host operands of ``dtype``. fp16 -> np.float16; bf16/fp8/bf8 require the ``ml_dtypes`` package (imported - lazily) and use FNUZ fp8 encodings for gfx942 parity. + lazily). The fp8/bf8 encoding follows the target arch: OCP (e4m3fn/e5m2) on + gfx950/gfx12, FNUZ (e4m3fnuz/e5m2fnuz) everywhere else. ``use_ocp=None`` + resolves from the local GPU; pass it explicitly when building for another arch. """ token = _DTYPE_ALIASES.get(str(dtype).lower()) if token is None: @@ -198,10 +212,11 @@ def numpy_dtype_for(dtype: str): ) from exc if token == "bf16": return np.dtype(ml_dtypes.bfloat16) + ocp = _resolve_use_ocp(use_ocp) if token == "fp8": - return np.dtype(ml_dtypes.float8_e4m3fnuz) + return np.dtype(ml_dtypes.float8_e4m3fn if ocp else ml_dtypes.float8_e4m3fnuz) if token == "bf8": - return np.dtype(ml_dtypes.float8_e5m2fnuz) + return np.dtype(ml_dtypes.float8_e5m2 if ocp else ml_dtypes.float8_e5m2fnuz) raise ValueError(f"Unsupported grouped GEMM dtype: {dtype!r}") # pragma: no cover @@ -225,14 +240,16 @@ def output_dtype_for(dtype: str) -> str: return CommonTypeMappings.get_output_dtype(token) -def output_numpy_dtype_for(dtype: str): +def output_numpy_dtype_for(dtype: str, use_ocp: Optional[bool] = None): """Numpy dtype of a kernel's OUTPUT buffer for input ``dtype``. Composition of :func:`output_dtype_for` + :func:`numpy_dtype_for`. For fp8/bf8 this resolves to ``np.float16`` (2 bytes) because the kernel's - ``CDataType`` is fp16; for fp16/bf16 it equals the input dtype. + ``CDataType`` is fp16; for fp16/bf16 it equals the input dtype. ``use_ocp`` + is therefore inert today, but is threaded through so an fp8-output kernel + would not silently pick the wrong encoding. """ - return numpy_dtype_for(output_dtype_for(dtype)) + return numpy_dtype_for(output_dtype_for(dtype), use_ocp=use_ocp) # ============================================================================ @@ -872,7 +889,7 @@ def _bf16_u16_to_fp32(u16: np.ndarray) -> np.ndarray: # --------------------------------------------------------------------------- -# fp8 (E4M3) / bf8 (E5M2) -- FNUZ ("NANOO") encoding used by gfx942/MI300. +# fp8 (E4M3) / bf8 (E5M2) -- both the FNUZ and the OCP encoding. # # numpy has no native 8-bit float, and the C ABI only cares about the 1-byte # memory layout (sizeof(fp8_t) == sizeof(bf8_t) == 1). We carry the value as a @@ -881,32 +898,79 @@ def _bf16_u16_to_fp32(u16: np.ndarray) -> np.ndarray: # NumPy reference multiplies bit-for-bit what the GPU multiplies. The ENCODE only # needs to land on the nearest representable byte. # -# FNUZ format (gfx942): bias = 2^(exp_bits-1); the all-1s exponent is a normal -# number (no Inf), the sole NaN is the sign=1/exp=0/mant=0 byte (0x80), and there -# is no negative zero. gfx950/MI350 uses the OCP fp8 format instead; this codec -# targets the gfx942 default and the OCP path needs separate handling. +# The two formats differ in exponent bias and in what the all-1s exponent means: +# +# FNUZ ("NANOO", gfx942 and every non-gfx950/gfx12 arch) +# bias = 2^(exp_bits-1) (8 / 16); the all-1s exponent is an ordinary normal +# number (no Inf); the sole NaN is 0x80 (sign=1, exp=0, mant=0), so there is +# no negative zero. Max finite: 240 (e4m3fnuz) / 57344 (e5m2fnuz). +# +# OCP (gfx950, gfx12) +# bias = 2^(exp_bits-1) - 1 (7 / 15); 0x80 is -0.0. e4m3fn has no Inf and +# reserves only exp=1111,mant=111 for NaN (max finite 448); e5m2 is IEEE-like, +# with Inf at exp=11111,mant=0 and NaN otherwise (max finite 57344). +# +# Which one applies is a property of the *target arch*, not of the header: the +# host pass of config.hpp always resolves to FNUZ (see dispatcher_common), so the +# selection must be made here, from the arch, and mirrored into the compile flags +# via ocp_arch_defines(). Getting this wrong does not crash -- it silently shifts +# every reference value by a factor of two. # --------------------------------------------------------------------------- +@functools.lru_cache(maxsize=1) +def _default_use_ocp() -> bool: + """Whether the host fp8/bf8 codec should emit OCP bytes for the local GPU. + + Falls back to FNUZ when no arch can be detected -- that is CK's default for + every arch outside gfx950/gfx12, and matches what a header compiled without + an explicit -DCK_TILE_USE_OCP_FP8 would pick on the host pass. + """ + try: + return _fp8_uses_ocp(_get_arch()) + except Exception: # noqa: BLE001 - no GPU / unknown arch + return False + + +def _resolve_use_ocp(use_ocp: Optional[bool]) -> bool: + return _default_use_ocp() if use_ocp is None else bool(use_ocp) + + @functools.lru_cache(maxsize=None) -def _fnuz_decode_table(exp_bits: int, mant_bits: int) -> np.ndarray: - """Build the 256-entry byte -> fp32 value table for an 8-bit FNUZ float. +def _fp8_decode_table(exp_bits: int, mant_bits: int, use_ocp: bool) -> np.ndarray: + """Build the 256-entry byte -> fp32 value table for an 8-bit float format. - The table is a pure function of (exp_bits, mant_bits), so it is cached; the - returned array is marked read-only because callers share the one instance. + The table is a pure function of (exp_bits, mant_bits, use_ocp), so it is + cached; the returned array is marked read-only because callers share the one + instance. """ - bias = (1 << (exp_bits - 1)) + bias = (1 << (exp_bits - 1)) - 1 if use_ocp else (1 << (exp_bits - 1)) mant_max = 1 << mant_bits sign_shift = exp_bits + mant_bits exp_mask = (1 << exp_bits) - 1 + # OCP e4m3fn has no Inf and steals only the all-1s mantissa for NaN; OCP e5m2 + # is IEEE-like. FNUZ has neither -- its all-1s exponent is just a big normal. + ocp_has_inf = use_ocp and mant_bits == 2 table = np.zeros(256, dtype=np.float32) for b in range(256): sign = (b >> sign_shift) & 1 exp = (b >> mant_bits) & exp_mask mant = b & (mant_max - 1) if exp == 0 and mant == 0: - # +0 (0x00); the negative-zero slot (0x80) is the lone NaN. - table[b] = np.float32(np.nan) if sign else np.float32(0.0) + if use_ocp: + table[b] = np.float32(-0.0 if sign else 0.0) + else: + # +0 (0x00); the negative-zero slot (0x80) is the lone NaN. + table[b] = np.float32(np.nan) if sign else np.float32(0.0) + continue + if use_ocp and exp == exp_mask: + if ocp_has_inf: + val = np.inf if mant == 0 else np.nan + elif mant == mant_max - 1: + val = np.nan + else: + val = (1.0 + mant / mant_max) * (2.0 ** (exp - bias)) + table[b] = np.float32(-val if (sign and val == val) else val) continue if exp == 0: val = (mant / mant_max) * (2.0 ** (1 - bias)) # subnormal @@ -917,16 +981,27 @@ def _fnuz_decode_table(exp_bits: int, mant_bits: int) -> np.ndarray: return table -def _fnuz_encode(x: np.ndarray, exp_bits: int, mant_bits: int) -> np.ndarray: - """Encode fp32 -> nearest 8-bit FNUZ float, returned as a uint8 bit pattern.""" - table = _fnuz_decode_table(exp_bits, mant_bits) +def _fnuz_decode_table(exp_bits: int, mant_bits: int) -> np.ndarray: + """FNUZ-only view of :func:`_fp8_decode_table` (kept for existing callers).""" + return _fp8_decode_table(exp_bits, mant_bits, False) + + +def _fp8_encode( + x: np.ndarray, exp_bits: int, mant_bits: int, use_ocp: Optional[bool] = None +) -> np.ndarray: + """Encode fp32 -> nearest 8-bit float, returned as a uint8 bit pattern.""" + use_ocp = _resolve_use_ocp(use_ocp) + table = _fp8_decode_table(exp_bits, mant_bits, use_ocp) sign_byte = np.uint8(1 << (exp_bits + mant_bits)) # 0x80 - # Positive half (bytes 0..127) holds every non-negative magnitude, sorted. + # Positive half (bytes 0..127) holds every non-negative magnitude. Drop the + # non-finite slots (OCP NaN/Inf) so out-of-range inputs saturate to the max + # finite value instead of rounding to Inf, matching CK's saturating convert. # Compare in float64: for very large inputs the gap between the two top # magnitudes is below fp32 resolution, which would tie and mis-saturate. pos_mag = table[: int(sign_byte)].astype(np.float64) - order = np.argsort(pos_mag) + finite = np.flatnonzero(np.isfinite(pos_mag)) + order = finite[np.argsort(pos_mag[finite])] sorted_mag = pos_mag[order] sorted_byte = order.astype(np.uint8) @@ -941,31 +1016,36 @@ def _fnuz_encode(x: np.ndarray, exp_bits: int, mant_bits: int) -> np.ndarray: chosen = np.where(pick_lo, lo, hi) out = sorted_byte[chosen] - # Apply sign, but never the 0x80 (-0 == NaN) slot: zeros stay +0. + # Apply sign. Under FNUZ 0x80 is the NaN slot rather than -0, so zeros must + # stay positive there; under OCP -0.0 is representable but +0 is equivalent + # for every consumer here, so keep the same rule for both. is_zero = sorted_mag[chosen] == 0 out = np.where((xf < 0) & ~is_zero, out | sign_byte, out) - out = np.where(np.isnan(xf), sign_byte, out) # NaN inputs -> NaN byte + # NaN byte: 0x80 under FNUZ, all-1s exponent and mantissa under OCP. + nan_byte = np.uint8(((1 << exp_bits) - 1) << mant_bits | ((1 << mant_bits) - 1)) \ + if use_ocp else sign_byte + out = np.where(np.isnan(xf), nan_byte, out) return out.astype(np.uint8).reshape(np.shape(x)) -def _fp32_to_fp8_u8(x: np.ndarray) -> np.ndarray: - """Encode fp32 -> fp8 E4M3 (FNUZ) bit pattern in a uint8 array.""" - return _fnuz_encode(x, exp_bits=4, mant_bits=3) +def _fp32_to_fp8_u8(x: np.ndarray, use_ocp: Optional[bool] = None) -> np.ndarray: + """Encode fp32 -> fp8 E4M3 bit pattern in a uint8 array (arch-selected format).""" + return _fp8_encode(x, exp_bits=4, mant_bits=3, use_ocp=use_ocp) -def _fp8_u8_to_fp32(u8: np.ndarray) -> np.ndarray: - """Decode an fp8 E4M3 (FNUZ) bit pattern back to fp32.""" - return _fnuz_decode_table(4, 3)[u8.astype(np.intp)] +def _fp8_u8_to_fp32(u8: np.ndarray, use_ocp: Optional[bool] = None) -> np.ndarray: + """Decode an fp8 E4M3 bit pattern back to fp32 (arch-selected format).""" + return _fp8_decode_table(4, 3, _resolve_use_ocp(use_ocp))[u8.astype(np.intp)] -def _fp32_to_bf8_u8(x: np.ndarray) -> np.ndarray: - """Encode fp32 -> bf8 E5M2 (FNUZ) bit pattern in a uint8 array.""" - return _fnuz_encode(x, exp_bits=5, mant_bits=2) +def _fp32_to_bf8_u8(x: np.ndarray, use_ocp: Optional[bool] = None) -> np.ndarray: + """Encode fp32 -> bf8 E5M2 bit pattern in a uint8 array (arch-selected format).""" + return _fp8_encode(x, exp_bits=5, mant_bits=2, use_ocp=use_ocp) -def _bf8_u8_to_fp32(u8: np.ndarray) -> np.ndarray: - """Decode a bf8 E5M2 (FNUZ) bit pattern back to fp32.""" - return _fnuz_decode_table(5, 2)[u8.astype(np.intp)] +def _bf8_u8_to_fp32(u8: np.ndarray, use_ocp: Optional[bool] = None) -> np.ndarray: + """Decode a bf8 E5M2 bit pattern back to fp32 (arch-selected format).""" + return _fp8_decode_table(5, 2, _resolve_use_ocp(use_ocp))[u8.astype(np.intp)] # Output (C) element dtype for an A/B element dtype, mirroring the codegen's @@ -1004,12 +1084,16 @@ class GpuGemmRunner: numpy arrays straight to the .so. """ - def __init__(self, lib_path: Path): + def __init__(self, lib_path: Path, arch: Optional[str] = None): self.lib = GemmDispatcherLib(lib_path) if not self.lib.initialize(): raise RuntimeError(f"Failed to initialize dispatcher .so: {lib_path}") names = self.lib.kernel_names self._kernel_name = names[0] if names else "unknown" + # fp8/bf8 encoding must match the arch the .so was compiled for. Running + # a .so requires the matching GPU, so the local arch is the right default; + # `arch` is the explicit pin for callers that already know it. + self._use_ocp = _fp8_uses_ocp(arch) if arch else None @property def kernel_name(self) -> str: @@ -1047,11 +1131,11 @@ def run( A_h = _fp32_to_bf16_u16(A_lay) B_h = _fp32_to_bf16_u16(B_lay) elif dtype == "fp8": - A_h = _fp32_to_fp8_u8(A_lay) - B_h = _fp32_to_fp8_u8(B_lay) + A_h = _fp32_to_fp8_u8(A_lay, use_ocp=self._use_ocp) + B_h = _fp32_to_fp8_u8(B_lay, use_ocp=self._use_ocp) elif dtype == "bf8": - A_h = _fp32_to_bf8_u8(A_lay) - B_h = _fp32_to_bf8_u8(B_lay) + A_h = _fp32_to_bf8_u8(A_lay, use_ocp=self._use_ocp) + B_h = _fp32_to_bf8_u8(B_lay, use_ocp=self._use_ocp) elif dtype == "int8": A_h = np.ascontiguousarray(A_lay, dtype=np.int8) B_h = np.ascontiguousarray(B_lay, dtype=np.int8) @@ -1111,7 +1195,13 @@ class GpuGroupedGemmRunner: overrun the host buffer (heap corruption). See :func:`output_numpy_dtype_for`. """ - def __init__(self, lib_path: Path, dtype: str = "fp16", layout: str = "rcr"): + def __init__( + self, + lib_path: Path, + dtype: str = "fp16", + layout: str = "rcr", + arch: Optional[str] = None, + ): self.lib = GemmDispatcherLib(lib_path) if not self.lib.initialize(): raise RuntimeError( @@ -1120,10 +1210,13 @@ def __init__(self, lib_path: Path, dtype: str = "fp16", layout: str = "rcr"): names = self.lib.kernel_names self._kernel_name = names[0] if names else "unknown" self._dtype = dtype + # fp8/bf8 encoding must match the arch the .so was compiled for; see + # GpuGemmRunner.__init__. None -> resolve from the local GPU. + self._use_ocp = _fp8_uses_ocp(arch) if arch else None # A/B (input) codec vs C (output) codec: they differ for fp8/bf8 # (output is fp16), so keep them distinct to size the C buffer correctly. - self._np_dtype = numpy_dtype_for(dtype) - self._c_np_dtype = output_numpy_dtype_for(dtype) + self._np_dtype = numpy_dtype_for(dtype, use_ocp=self._use_ocp) + self._c_np_dtype = output_numpy_dtype_for(dtype, use_ocp=self._use_ocp) if len(layout) != 3 or any(ch not in ("r", "c") for ch in layout): raise ValueError(f"layout must be a 3-char r/c string, got {layout!r}") self._layout = layout @@ -1862,6 +1955,12 @@ def _build_compile_jobs( "-D__HIP_PLATFORM_AMD__", f"--offload-arch={config.gfx_arch}", f'-DGFX_ARCH="{config.gfx_arch}"', + # Pin the fp8/bf8 encoding so BOTH compiler passes agree. Without this the + # device pass of config.hpp sees __gfx950__ and picks OCP while the host + # pass falls back to FNUZ -- and the numpy reference, which follows the + # arch (see numpy_dtype_for), then disagrees with the kernel by a factor + # of two. FNUZ archs need no define; the list is empty for them. + *_ocp_arch_defines(config.gfx_arch), # Match Tile Engine's AMDGPU codegen flags exactly (see variant_flags / # _tile_engine_codegen_flags). Without them the kernel is compiled with # different inlining/register allocation, which changes occupancy; diff --git a/projects/composablekernel/dispatcher/python/grouped_gemm_abquant_utils.py b/projects/composablekernel/dispatcher/python/grouped_gemm_abquant_utils.py index a55458104fac..440a74e5bbb8 100644 --- a/projects/composablekernel/dispatcher/python/grouped_gemm_abquant_utils.py +++ b/projects/composablekernel/dispatcher/python/grouped_gemm_abquant_utils.py @@ -54,6 +54,11 @@ sys.path.insert(0, _codegen_dir) from codegen_common import make_abquant_kernel_name # noqa: E402 +_python_dir = str(Path(__file__).parent) +if _python_dir not in sys.path: + sys.path.insert(0, _python_dir) +from dispatcher_common import arch_feature_defines # noqa: E402 + _DEFAULT_HIPCC = "hipcc" _DEFAULT_GFX_ARCH = "gfx950" @@ -546,9 +551,16 @@ def _compile_abquant_kernel( ) -> bool: ck_include = _get_ck_include_dir() + # Arch-specific defines: gfx950 uses OCP fp8 (not FNUZ) and native MX support. + # These mirror the CMakeLists.txt definitions that are normally injected by CMake + # but are absent in the standalone hipcc build path. Without them the host pass of + # config.hpp silently falls back to FNUZ while the device pass picks OCP. + arch_defines = arch_feature_defines(gfx_arch) + cmd = [hipcc] + _HIPCC_BASE_FLAGS + [ f"--offload-arch={gfx_arch}", f"-DGFX_ARCH=\"{gfx_arch}\"", + *arch_defines, "-include", str(hpp_path), str(_CTYPES_LIB_SRC), "-o", str(so_path), diff --git a/projects/composablekernel/dispatcher/python/grouped_gemm_aquant_utils.py b/projects/composablekernel/dispatcher/python/grouped_gemm_aquant_utils.py index f36524c780ea..2436e450a171 100644 --- a/projects/composablekernel/dispatcher/python/grouped_gemm_aquant_utils.py +++ b/projects/composablekernel/dispatcher/python/grouped_gemm_aquant_utils.py @@ -54,6 +54,11 @@ sys.path.insert(0, _codegen_dir) from codegen_common import make_aquant_kernel_name # noqa: E402 +_python_dir = str(Path(__file__).parent) +if _python_dir not in sys.path: + sys.path.insert(0, _python_dir) +from dispatcher_common import arch_feature_defines # noqa: E402 + _DEFAULT_HIPCC = "hipcc" _DEFAULT_GFX_ARCH = "gfx950" @@ -500,13 +505,8 @@ def _compile_aquant_kernel( # Arch-specific defines: gfx950 uses OCP fp8 (not FNUZ) and native MX support. # These mirror the CMakeLists.txt definitions that are normally injected by CMake - # but are absent in the standalone hipcc build path. Same block as - # grouped_gemm_bquant_utils._compile_bquant_kernel. - arch_defines = [] - if "gfx12" in gfx_arch or "gfx950" in gfx_arch: - arch_defines += ["-DCK_USE_OCP_FP8", "-DCK_TILE_USE_OCP_FP8"] - if "gfx950" in gfx_arch: - arch_defines += ["-DCK_USE_NATIVE_MX_SUPPORT", "-DCK_GFX950_SUPPORT"] + # but are absent in the standalone hipcc build path. + arch_defines = arch_feature_defines(gfx_arch) cmd = [hipcc] + _HIPCC_BASE_FLAGS + [ f"--offload-arch={gfx_arch}", diff --git a/projects/composablekernel/dispatcher/python/grouped_gemm_bquant_utils.py b/projects/composablekernel/dispatcher/python/grouped_gemm_bquant_utils.py index 92fb0038c8f7..f75753630175 100644 --- a/projects/composablekernel/dispatcher/python/grouped_gemm_bquant_utils.py +++ b/projects/composablekernel/dispatcher/python/grouped_gemm_bquant_utils.py @@ -52,6 +52,11 @@ sys.path.insert(0, _codegen_dir) from codegen_common import make_bquant_kernel_name # noqa: E402 +_python_dir = str(Path(__file__).parent) +if _python_dir not in sys.path: + sys.path.insert(0, _python_dir) +from dispatcher_common import arch_feature_defines # noqa: E402 + _DEFAULT_HIPCC = "hipcc" _DEFAULT_GFX_ARCH = "gfx950" @@ -524,11 +529,7 @@ def _compile_bquant_kernel( # Arch-specific defines: gfx950 uses OCP fp8 (not FNUZ) and native MX support. # These mirror the CMakeLists.txt definitions that are normally injected by CMake # but are absent in the standalone hipcc build path. - arch_defines = [] - if "gfx12" in gfx_arch or "gfx950" in gfx_arch: - arch_defines += ["-DCK_USE_OCP_FP8", "-DCK_TILE_USE_OCP_FP8"] - if "gfx950" in gfx_arch: - arch_defines += ["-DCK_USE_NATIVE_MX_SUPPORT", "-DCK_GFX950_SUPPORT"] + arch_defines = arch_feature_defines(gfx_arch) compile_cmd = [hipcc, "-c", "-fPIC", "-O3", "-std=c++17", "-DCK_TILE_SINGLE_KERNEL_INCLUDE", "-w", diff --git a/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py b/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py index 7bb7d1338770..34a086480b06 100644 --- a/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py +++ b/projects/composablekernel/dispatcher/python/grouped_gemm_rowcolquant_utils.py @@ -54,6 +54,11 @@ make_rowcolquant_kernel_name, ) +_python_dir = str(Path(__file__).parent) +if _python_dir not in sys.path: + sys.path.insert(0, _python_dir) +from dispatcher_common import arch_feature_defines # noqa: E402 + _DEFAULT_HIPCC = "hipcc" _DEFAULT_GFX_ARCH = "gfx950" @@ -500,11 +505,7 @@ def _compile_rowcolquant_kernel( obj_path = so_path.with_suffix(".o") - arch_defines = [] - if "gfx12" in gfx_arch or "gfx950" in gfx_arch: - arch_defines += ["-DCK_USE_OCP_FP8", "-DCK_TILE_USE_OCP_FP8"] - if "gfx950" in gfx_arch: - arch_defines += ["-DCK_USE_NATIVE_MX_SUPPORT", "-DCK_GFX950_SUPPORT"] + arch_defines = arch_feature_defines(gfx_arch) compile_cmd = [hipcc, "-c", "-fPIC", "-O3", "-std=c++17", "-DCK_TILE_SINGLE_KERNEL_INCLUDE", "-w", diff --git a/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py b/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py index e240b0f9604a..ef3f02e33b42 100644 --- a/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py +++ b/projects/composablekernel/dispatcher/python/grouped_gemm_tensorquant_utils.py @@ -54,6 +54,11 @@ make_tensorquant_kernel_name, ) +_python_dir = str(Path(__file__).parent) +if _python_dir not in sys.path: + sys.path.insert(0, _python_dir) +from dispatcher_common import arch_feature_defines # noqa: E402 + _DEFAULT_HIPCC = "hipcc" _DEFAULT_GFX_ARCH = "gfx950" @@ -488,11 +493,7 @@ def _compile_tensorquant_kernel( obj_path = so_path.with_suffix(".o") - arch_defines = [] - if "gfx12" in gfx_arch or "gfx950" in gfx_arch: - arch_defines += ["-DCK_USE_OCP_FP8", "-DCK_TILE_USE_OCP_FP8"] - if "gfx950" in gfx_arch: - arch_defines += ["-DCK_USE_NATIVE_MX_SUPPORT", "-DCK_GFX950_SUPPORT"] + arch_defines = arch_feature_defines(gfx_arch) compile_cmd = [hipcc, "-c", "-fPIC", "-O3", "-std=c++17", "-DCK_TILE_SINGLE_KERNEL_INCLUDE", "-w", diff --git a/projects/composablekernel/dispatcher/tests/test_gemm_parity.py b/projects/composablekernel/dispatcher/tests/test_gemm_parity.py index 0814e11ee650..8ed4325acbcb 100644 --- a/projects/composablekernel/dispatcher/tests/test_gemm_parity.py +++ b/projects/composablekernel/dispatcher/tests/test_gemm_parity.py @@ -89,9 +89,15 @@ # Global-relative-error gates. fp16 measured ~3-4e-4 and bf16 ~8e-3 on gfx942. # fp8/bf8 are far coarser (3- and 2-bit mantissa) so their gates are looser; int8 -# is an exact integer accumulation so it must match bit-for-bit. The fp8/bf8 -# gates are first-cut headroom values and may want tightening once measured on a -# GPU. +# is an exact integer accumulation so it must match bit-for-bit. +# +# The fp8/bf8 numbers are still first-cut headroom, and they are deliberately +# unchanged by the OCP/FNUZ fix. That fix removed a real source of error on +# gfx950 -- the reference used to be FNUZ while the kernel ran OCP, a factor-of- +# two shift these gates were wide enough to swallow -- but it changed nothing on +# gfx942, which was FNUZ-correct all along and still needs this much room purely +# for 3-/2-bit quantization. Tightening is a measurement, not a deduction: run +# the sweep on both archs and set each gate from the observed maximum. _TOL = { "fp16": 2e-3, "bf16": 1.5e-2, diff --git a/projects/composablekernel/dispatcher/tests/test_gemm_utils.py b/projects/composablekernel/dispatcher/tests/test_gemm_utils.py index 14afc24a2b19..6622bd385cee 100644 --- a/projects/composablekernel/dispatcher/tests/test_gemm_utils.py +++ b/projects/composablekernel/dispatcher/tests/test_gemm_utils.py @@ -37,6 +37,9 @@ _fp32_to_bf8_u8, _bf8_u8_to_fp32, _fnuz_decode_table, + _fp8_decode_table, + _fp8_encode, + numpy_dtype_for, _output_dtype, _dtype_from_kernel_name, _layout_from_kernel_name, @@ -44,6 +47,19 @@ ) +def _require_ml_dtypes(): + """Return the ``ml_dtypes`` module, skipping the test when it is absent. + + It is the independent oracle for the fp8/bf8 tables, so a test that needs it + must skip rather than silently assert nothing. + """ + try: + import ml_dtypes # noqa: WPS433 (lazy: optional dep) + except ImportError: # pragma: no cover - env-dependent + raise unittest.SkipTest("ml_dtypes is required for fp8/bf8 table checks") + return ml_dtypes + + class TestBf16Encoding(unittest.TestCase): """bf16 = top 16 bits of fp32 with round-to-nearest-even.""" @@ -129,10 +145,14 @@ def test_decode_is_consistent_with_encode(self): self.assertTrue(np.all(np.isfinite(d))) def test_saturates_no_inf(self): - # FNUZ has no infinity: huge magnitudes clamp to the finite max. + # FNUZ has no infinity: huge magnitudes clamp to the finite max. Pin the + # format explicitly -- the wrappers otherwise follow the local GPU, and on + # a gfx950 runner these are the OCP maxima (448 / -57344) instead. big = np.array([1e30, -1e30], dtype=np.float32) - self.assertEqual(float(_fp8_u8_to_fp32(_fp32_to_fp8_u8(big))[0]), 240.0) - self.assertEqual(float(_bf8_u8_to_fp32(_fp32_to_bf8_u8(big))[1]), -57344.0) + enc8 = _fp32_to_fp8_u8(big, use_ocp=False) + enc5 = _fp32_to_bf8_u8(big, use_ocp=False) + self.assertEqual(float(_fp8_u8_to_fp32(enc8, use_ocp=False)[0]), 240.0) + self.assertEqual(float(_bf8_u8_to_fp32(enc5, use_ocp=False)[1]), -57344.0) def test_dtype_and_size(self): for enc in (_fp32_to_fp8_u8, _fp32_to_bf8_u8): @@ -141,6 +161,100 @@ def test_dtype_and_size(self): self.assertEqual(u8.itemsize, 1) # must match sizeof(fp8_t/bf8_t) +class TestFp8Bf8OcpEncoding(unittest.TestCase): + """The OCP (gfx950/gfx12) half of the fp8/bf8 codec. + + The decode table is the load-bearing half -- it defines what the numpy + reference multiplies -- so it is checked byte-for-byte against ml_dtypes + rather than spot-checked. Getting the format wrong does not crash; it + silently rescales every reference value by a factor of two. + """ + + def test_decode_matches_ml_dtypes_for_every_byte(self): + ml_dtypes = _require_ml_dtypes() + cases = ( + (4, 3, True, ml_dtypes.float8_e4m3fn), + (4, 3, False, ml_dtypes.float8_e4m3fnuz), + (5, 2, True, ml_dtypes.float8_e5m2), + (5, 2, False, ml_dtypes.float8_e5m2fnuz), + ) + all_bytes = np.arange(256, dtype=np.uint8) + for exp_bits, mant_bits, use_ocp, ml_type in cases: + with self.subTest(dtype=ml_type.__name__): + ours = _fp8_decode_table(exp_bits, mant_bits, use_ocp) + theirs = all_bytes.view(ml_type).astype(np.float32) + same = (ours == theirs) | (np.isnan(ours) & np.isnan(theirs)) + self.assertTrue( + np.all(same), + f"bytes {np.flatnonzero(~same).tolist()} disagree with ml_dtypes", + ) + + def test_ocp_format_ranges(self): + # OCP bias is one less than FNUZ, so e4m3fn reaches 448 (not 240); + # e5m2 keeps 57344 but gains a real infinity. + t43 = _fp8_decode_table(4, 3, True) + t52 = _fp8_decode_table(5, 2, True) + self.assertEqual(float(np.nanmax(t43[np.isfinite(t43)])), 448.0) + self.assertEqual(float(np.nanmin(t43[np.isfinite(t43)])), -448.0) + self.assertEqual(float(np.nanmax(t52[np.isfinite(t52)])), 57344.0) + self.assertEqual(np.count_nonzero(np.isinf(t43)), 0) # e4m3fn has no Inf + self.assertEqual(np.count_nonzero(np.isinf(t52)), 2) # e5m2 does + + def test_ocp_zero_slots(self): + # Unlike FNUZ, 0x80 is a genuine -0.0 rather than the lone NaN. + for tab in (_fp8_decode_table(4, 3, True), _fp8_decode_table(5, 2, True)): + self.assertEqual(float(tab[0x00]), 0.0) + self.assertFalse(np.isnan(tab[0x80])) + self.assertEqual(float(tab[0x80]), 0.0) + + def test_every_representable_value_roundtrips_exactly(self): + for exp_bits, mant_bits in ((4, 3), (5, 2)): + for use_ocp in (False, True): + with self.subTest(bits=(exp_bits, mant_bits), ocp=use_ocp): + table = _fp8_decode_table(exp_bits, mant_bits, use_ocp) + vals = table[np.isfinite(table)] + enc = _fp8_encode(vals, exp_bits, mant_bits, use_ocp=use_ocp) + np.testing.assert_array_equal(table[enc.astype(np.intp)], vals) + + def test_ocp_saturates_to_max_finite_not_inf(self): + # e5m2 has an Inf byte; a naive nearest-neighbour search would round + # huge inputs to it. CK's convert saturates, so the codec must too. + big = np.array([1e30, -1e30], dtype=np.float32) + table = _fp8_decode_table(5, 2, True) + got = table[_fp8_encode(big, 5, 2, use_ocp=True).astype(np.intp)] + self.assertEqual(float(got[0]), 57344.0) + self.assertEqual(float(got[1]), -57344.0) + t43 = _fp8_decode_table(4, 3, True) + got8 = t43[_fp8_encode(big, 4, 3, use_ocp=True).astype(np.intp)] + self.assertEqual(float(got8[0]), 448.0) + + def test_nan_maps_to_a_nan_byte_in_both_formats(self): + nan = np.array([np.nan], dtype=np.float32) + for exp_bits, mant_bits in ((4, 3), (5, 2)): + for use_ocp in (False, True): + with self.subTest(bits=(exp_bits, mant_bits), ocp=use_ocp): + table = _fp8_decode_table(exp_bits, mant_bits, use_ocp) + enc = _fp8_encode(nan, exp_bits, mant_bits, use_ocp=use_ocp) + self.assertTrue(np.isnan(table[int(enc[0])])) + + def test_fnuz_aliases_forward_to_the_shared_table(self): + # _fnuz_decode_table is kept as a thin FNUZ view; callers rely on the + # lru_cache identity, so it must return the cached instance itself. + self.assertIs(_fnuz_decode_table(4, 3), _fp8_decode_table(4, 3, False)) + self.assertIsNot(_fnuz_decode_table(4, 3), _fp8_decode_table(4, 3, True)) + + def test_numpy_dtype_follows_the_requested_format(self): + ml_dtypes = _require_ml_dtypes() + self.assertEqual(numpy_dtype_for("fp8", use_ocp=True), + np.dtype(ml_dtypes.float8_e4m3fn)) + self.assertEqual(numpy_dtype_for("fp8", use_ocp=False), + np.dtype(ml_dtypes.float8_e4m3fnuz)) + self.assertEqual(numpy_dtype_for("bf8", use_ocp=True), + np.dtype(ml_dtypes.float8_e5m2)) + self.assertEqual(numpy_dtype_for("bf8", use_ocp=False), + np.dtype(ml_dtypes.float8_e5m2fnuz)) + + class TestOutputDtype(unittest.TestCase): """Output (C) element dtype must mirror the codegen's get_output_dtype.""" diff --git a/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py index b487276f46d0..4b36c6e34d16 100644 --- a/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py @@ -32,6 +32,8 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "python")) +from dispatcher_common import fp8_uses_ocp as _fp8_uses_ocp + from grouped_gemm_rowcolquant_utils import ( RowColQuantGemmProblem, RowColQuantGpuGemmRunner, @@ -100,17 +102,6 @@ def _require_ml_dtypes(): ) -def _fp8_uses_ocp(arch: str) -> bool: - """Mirror the -DCK_USE_OCP_FP8 compile logic in grouped_gemm_rowcolquant_utils.py. - - gfx950 / gfx12 build the kernel with OCP fp8 (e4m3fn / e5m2); every other - supported arch (notably gfx942) uses the native FNUZ format - (e4m3fnuz / e5m2fnuz). The host-side encoding MUST match the format the kernel - was compiled for, otherwise the reinterpreted bytes decode to NaN/Inf on device. - """ - return "gfx950" in arch or "gfx12" in arch - - def _fp8_ml_dtype(dtype: str): """Return the ml_dtypes fp8 type matching the compiled kernel's arch.""" _require_ml_dtypes() diff --git a/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py index dcd54512ae31..9d797acfc290 100644 --- a/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py @@ -32,6 +32,8 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "python")) +from dispatcher_common import fp8_uses_ocp as _fp8_uses_ocp + from grouped_gemm_tensorquant_utils import ( TensorQuantGemmProblem, TensorQuantGpuGemmRunner, @@ -100,17 +102,6 @@ def _require_ml_dtypes(): ) -def _fp8_uses_ocp(arch: str) -> bool: - """Mirror the -DCK_USE_OCP_FP8 compile logic in grouped_gemm_tensorquant_utils.py. - - gfx950 / gfx12 build the kernel with OCP fp8 (e4m3fn / e5m2); every other - supported arch (notably gfx942) uses the native FNUZ format - (e4m3fnuz / e5m2fnuz). The host-side encoding MUST match the format the kernel - was compiled for, otherwise the reinterpreted bytes decode to NaN/Inf on device. - """ - return "gfx950" in arch or "gfx12" in arch - - def _fp8_ml_dtype(dtype: str): """Return the ml_dtypes fp8 type matching the compiled kernel's arch.""" _require_ml_dtypes() diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 536155945cef..9236a88cf552 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1601,33 +1601,27 @@ def dispatcherGemmVariantsFor(String arch) { // Which dtypes the GEMM sweeps may ask for on each arch. // -// gfx950 is fp16/bf16 only, and the reason is not that the hardware lacks fp8 -- -// it is that the dispatcher's fp8 path is gfx942-shaped on both ends: +// gfx950 used to be fp16/bf16 only. Not because the hardware lacks fp8, but +// because the dispatcher's fp8 path was gfx942-shaped on both ends: the JIT +// handed hipcc only --offload-arch, so gfx950 kernels fell into the FNUZ #else +// branch of ck_tile/core/numeric/float8.hpp on the host pass while the device +// pass and the fp8 MFMA used OCP -- and the numpy reference in gemm_utils.py was +// unconditionally FNUZ. Three encodings, one test; the strata passed only because +// the parity gates are loose. Both halves are fixed now: // -// * Device: CMakeLists.txt (see the CK_TILE_USE_OCP_FP8 block) compiles gfx950 -// and gfx12 with -DCK_TILE_USE_OCP_FP8. The quant bridges replicate that in -// their own JIT flags (grouped_gemm_{a,b,rowcol,tensor}quant_utils.py all -// append -DCK_USE_OCP_FP8 -DCK_TILE_USE_OCP_FP8 for gfx950), but the GEMM -// sweep path does not: gemm_utils.py and ctypes_utils.py hand hipcc only -// --offload-arch. So a gfx950 node here JIT-builds kernels that fall into -// the FNUZ #else branch of ck_tile/core/numeric/float8.hpp, while gfx950's -// fp8 MFMA and its __gfx950__ conversion intrinsics are OCP. That asymmetry -// is why the quant operators keep their fp8 coverage on gfx950 and the -// sweeps below lose theirs. -// * Host: the fp8/bf8 codec in gemm_utils.py is FNUZ-only and says so -- -// "gfx950/MI350 uses the OCP fp8 format instead; this codec targets the -// gfx942 default and the OCP path needs separate handling." +// * Device: gemm_utils._build_compile_jobs appends ocp_arch_defines(gfx_arch) +// (-DCK_USE_OCP_FP8 -DCK_TILE_USE_OCP_FP8), matching what CMake does for a +// gfx950 build and what the quant bridges already did for their own JIT. +// * Host: the gemm_utils fp8/bf8 codec is arch-aware -- _fp8_decode_table +// carries both formats and is verified byte-for-byte against ml_dtypes, and +// numpy_dtype_for picks e4m3fn/e5m2 vs e4m3fnuz/e5m2fnuz from the arch. // -// Those strata therefore validate an encoding production never ships, and they -// pass only because the parity tolerances are loose (fp8 1.5e-1, bf8 3.0e-1). -// test_streamk_gpu_correctness.py already refuses gfx950 for exactly this -// reason. Re-widening this list is not the fix; the fix is to port -// test_rowcolquant_gpu_correctness.py's _fp8_uses_ocp() into gemm_utils AND to -// pass -DCK_TILE_USE_OCP_FP8 in the JIT flags. Both, or neither works. +// dispatcher_common.fp8_uses_ocp is the single predicate behind both, so they +// cannot drift apart again. def dispatcherSweepDtypesFor(String arch) { switch (arch) { case "gfx942": return "fp16,bf16,fp8,bf8" - case "gfx950": return "fp16,bf16" + case "gfx950": return "fp16,bf16,fp8,bf8" default: return "fp16,bf16" } } From 808d6c4a5c5eaf979d9d8b9a5ba2c0ac08109da9 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Wed, 26 Aug 2026 16:13:02 -0400 Subject: [PATCH 29/51] ci(ck-tile): run Stream-K on gfx950 Stream-K was pinned to gfx942 in three places, all of them attributing it to the "TODO: Add gfx950 when supported" in the Stream-K CMakeLists. That was the wrong cause. The dispatcher lane never touches those AOT targets -- test_gemm_search_space drives gemm_utils, which JIT-builds each config with hipcc and --offload-arch, and neither gemm_utils nor the sweep tables carry a stream_k arch guard anywhere. The actual blocker was the fp8/bf8 encoding mismatch on gfx950, which test_streamk_gpu_correctness said outright in its own comment, and which is now fixed. So: * SUPPORTED_ARCHS gains gfx950. gfx90a stays out for a real reason -- it has no fp8/bf8 MFMA, so half the dtype matrix is unrunnable there. * dispatcherGemmVariantsFor("gfx950") gains stream_k. * The two companion tests move out of their arch == "gfx942" branch and key off the variant list instead, so the sweep and its companions cannot disagree about which archs run Stream-K. The gate still matters: test_streamk_registry.py has no arch allow-list of its own. The CMake target list is widened to match every sibling GEMM op, none of which excludes gfx950. That part is AOT hygiene rather than a test unblocker, and it is the one change here not exercised locally -- this box is gfx90a, so a gfx950 AOT build has to confirm it in CI. If it turns out to surface unrelated failures it can be reverted on its own without affecting the lane. --- .../tests/test_streamk_gpu_correctness.py | 10 +++++--- .../composablekernel/groovy/vars/ck.groovy | 25 +++++++++++++------ .../ops/gemm_streamk/CMakeLists.txt | 4 +-- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/projects/composablekernel/dispatcher/tests/test_streamk_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_streamk_gpu_correctness.py index 0790c432c827..1ee73a1d0c40 100644 --- a/projects/composablekernel/dispatcher/tests/test_streamk_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_streamk_gpu_correctness.py @@ -68,10 +68,12 @@ PASS, FAIL = "PASS", "FAIL" SKIP_EXIT = 77 -# gfx942 only. gfx90a has no fp8/bf8 MFMA, and on gfx950 the fp8/bf8 host codecs -# in gemm_utils (FNUZ) disagree with the arch's OCP encoding — either arch would -# make half the matrix red for reasons unrelated to Stream-K. -SUPPORTED_ARCHS = ("gfx942",) +# gfx90a stays out: it has no fp8/bf8 MFMA, so half the matrix below is +# unrunnable there. gfx950 used to be excluded for a different reason -- the +# gemm_utils host codec was FNUZ-only while the arch encodes fp8 as OCP -- which +# had nothing to do with Stream-K. That codec now follows the arch (see +# dispatcher_common.fp8_uses_ocp), so gfx950 is back in. +SUPPORTED_ARCHS = ("gfx942", "gfx950") STRATEGIES = ("atomic", "linear", "tree") ALL_DTYPES = ("fp16", "bf16", "fp8", "bf8") diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 9236a88cf552..ed7415b78dec 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1577,10 +1577,16 @@ def runDispatcherPerfTests(String compiler, String gpuTarget = "gfx942") { } } -// Which GEMM variants of test_gemm_search_space.py each arch can run. Bounded by -// the tile_engine CMake arch guards -- the dispatcher cannot cover more archs -// than the kernels it selects are built for. stream_k is gfx942-only until the -// "TODO: add gfx950" in the stream-K CMakeLists is resolved. +// Which GEMM variants of test_gemm_search_space.py each arch can run. +// +// stream_k used to be gfx942-only here, attributed to the "TODO: add gfx950" in +// tile_engine/ops/gemm_streamk/CMakeLists.txt. That was the wrong cause: this +// path never touches those AOT targets -- test_gemm_search_space drives +// gemm_utils, which JIT-builds each config with hipcc and --offload-arch, and +// neither gemm_utils nor the sweep tables carry any stream_k arch guard. The +// real blocker was the fp8/bf8 encoding mismatch on gfx950, which is fixed (see +// dispatcherSweepDtypesFor below). The CMake TODO is still worth resolving, but +// as AOT hygiene, not as a gate on this lane. // // gfx1201 is absent on purpose, and the Jenkinsfile has no gfx1201 dispatcher // stage for the same reason: gemm_utils._SUPPORTED_ARCHES is @@ -1594,7 +1600,7 @@ def runDispatcherPerfTests(String compiler, String gpuTarget = "gfx942") { def dispatcherGemmVariantsFor(String arch) { switch (arch) { case "gfx942": return ["grouped", "multi_d", "multi_abd", "stream_k"] - case "gfx950": return ["grouped", "multi_d", "multi_abd"] + case "gfx950": return ["grouped", "multi_d", "multi_abd", "stream_k"] default: return [] } } @@ -1788,8 +1794,13 @@ def runDispatcherCorrectnessTests(String arch, String compiler) { // shared sweep above only ever builds the atomic reduction strategy // (default_ci_config.json has no streamk_config, so expand_sweep falls // back to ["atomic"]), leaving linear/tree unverified through ctypes. - // It takes --gfx and self-gates to gfx942. - if (arch == "gfx942") { + // It takes --gfx and self-gates via SUPPORTED_ARCHS, now (gfx942, gfx950). + // + // Both are keyed off the variant list rather than a hardcoded arch == "gfx942" + // branch, so the sweep and its companions can never disagree about which + // archs run Stream-K. test_streamk_registry.py has no arch allow-list of its + // own, which is precisely why it needs the gate here. + if (dispatcherGemmVariantsFor(arch).contains("stream_k")) { execute_cmd += """ && \ run_ok python3 ../dispatcher/tests/test_streamk_registry.py --arch ${arch} --datatypes fp16 --layouts rcr && \ run_ok python3 ../dispatcher/tests/test_streamk_gpu_correctness.py --gfx ${arch}""" diff --git a/projects/composablekernel/tile_engine/ops/gemm_streamk/CMakeLists.txt b/projects/composablekernel/tile_engine/ops/gemm_streamk/CMakeLists.txt index 02107de3fb8c..ca1ccff2c049 100644 --- a/projects/composablekernel/tile_engine/ops/gemm_streamk/CMakeLists.txt +++ b/projects/composablekernel/tile_engine/ops/gemm_streamk/CMakeLists.txt @@ -231,9 +231,9 @@ message(VERBOSE "GEMM_STREAMK_DATATYPE: ${GEMM_STREAMK_DATATYPE}") message(VERBOSE "GEMM_STREAMK_LAYOUT: ${GEMM_STREAMK_LAYOUT}") message(VERBOSE "SUPPORTED_GPU_TARGETS: ${SUPPORTED_GPU_TARGETS}") -# Filter GPU targets to only gfx90a, gfx942 +# Filter GPU targets to only gfx90a, gfx942, gfx950 set(GEMM_GPU_TARGETS_INDIVIDUAL "") -set(DESIRED_TARGETS "gfx90a;gfx942") # TODO: Add gfx950 when supported +set(DESIRED_TARGETS "gfx90a;gfx942;gfx950") foreach(target IN LISTS SUPPORTED_GPU_TARGETS) if(target IN_LIST DESIRED_TARGETS) From 0dce8b6519866a088034e54556fe709f70057481 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Wed, 26 Aug 2026 16:15:51 -0400 Subject: [PATCH 30/51] test(ck-tile): exercise all four Multi-D GEMM layouts Multi-D advertised four layouts and ran one. Both the sweep and the bridge-level GPU test were pinned to rcrr, so rrrr / ccrr / crrr were never built, let alone checked numerically. Nothing in the stack was actually blocking them: the codegen and ctypes lib are layout-generic, and GpuMultiDGemmRunner already reads the 4-char layout back out of the kernel name and transposes the host operands itself. A and B are handed over as logical (M, K) and (K, N) either way, which is why the fp32 reference needs no per-layout construction and why the diff is this small. test_gemm_search_space now defaults multi_d to all four. That costs no extra runtime -- _sample() allocates floor(budget / n_strata) per stratum, so a fixed --budget is redistributed rather than multiplied -- and all four enumerate 16 configs against default_ci_config.json, so none of the new strata can trip the runner's "no configs to run" exit. test_multi_d_gpu_correctness loops over the four with subTest, so one broken layout names itself instead of masking the other three. The existing all-zero and finiteness guards now run per layout, which matters more here than for the sweep: a layout wired up wrong is far more likely to produce quiet garbage than to fail to build. multi_abd stays single-layout until its bridge is verified the same way. --- .../tests/test_gemm_search_space.py | 11 ++++- .../tests/test_multi_d_gpu_correctness.py | 46 +++++++++++++++---- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py b/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py index 10f9acbe3826..12fd6057c3bc 100644 --- a/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py +++ b/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py @@ -73,12 +73,19 @@ # Per-variant defaults for --dtypes / --layouts. multi_abd and multi_d are # fp16-only end-to-end (codegen, ctypes lib and runner all assume fp16) and take -# the 4-char (A,B,C/E,D) layout code rather than the 3-char one. +# the 4-char (A,B,C/E,D) layout code rather than the 3-char one. Only the A and +# B chars vary -- the TE epilogue writes C/E and reads D row-major. +# +# Widening multi_d from one layout to four costs no extra runtime: _sample() +# allocates floor(budget / n_strata) configs per stratum, so a fixed --budget is +# redistributed rather than multiplied. multi_abd stays single-layout until its +# bridge is verified the same way multi_d now is. +_MULTI_LAYOUTS = ["rcrr", "rrrr", "ccrr", "crrr"] _VARIANT_DEFAULTS = { "standard": (_DTYPES, _LAYOUTS), "grouped": (_DTYPES, _LAYOUTS), "stream_k": (_DTYPES, _LAYOUTS), - "multi_d": (["fp16"], ["rcrr"]), + "multi_d": (["fp16"], _MULTI_LAYOUTS), "multi_abd": (["fp16"], ["rcrr"]), } _VARIANTS = tuple(_VARIANT_DEFAULTS) diff --git a/projects/composablekernel/dispatcher/tests/test_multi_d_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_multi_d_gpu_correctness.py index 7632d5321571..0913173471b2 100644 --- a/projects/composablekernel/dispatcher/tests/test_multi_d_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_multi_d_gpu_correctness.py @@ -13,7 +13,8 @@ Real numeric check -- random A/B/D, element-wise validation vs. fp32 reference, plus a non-zero / finite guard so a mis-launched (all-zero) kernel FAILS. -The multi_d TE op is fp16-only, so only fp16 is exercised. +The multi_d TE op is fp16-only, so only fp16 is exercised, but all four A/B +layout combinations are (rcrr / rrrr / ccrr / crrr). Runs green on gfx942 (MI300X). SKIPs cleanly with no GPU / hipcc / static lib. @@ -42,6 +43,12 @@ _TOL = 1e-2 # fp16 GEMM + fp16 D-fuse precision band +# 4-char (A, B, C/E, D) layout codes, matching _VARIANT_DEFAULTS["multi_d"] in +# test_gemm_search_space.py. C and E are row-major throughout: the TE multi_d +# epilogue writes row-major, so only the A and B chars actually vary. +_LAYOUTS = ("rcrr", "rrrr", "ccrr", "crrr") +_LAYOUT_WORD = {"r": "row", "c": "col"} + def _detect_arch(): # subprocess.run with a timeout (rocminfo can hang on misconfigured ROCm @@ -95,23 +102,37 @@ def setUp(self): self.skipTest("hipcc not found") def test_fp16_multid_add(self): + """E = A@B + D0 + D1 across all four A/B layout combinations. + + The runner reads the layout back out of the kernel name and transposes + the host operands itself, so A and B are handed over as logical (M, K) + and (K, N) in every case and the reference below is layout-independent. + Each layout is a separate build, hence subTest rather than four methods: + a failure names the layout without hiding the other three. + """ + for layout in _LAYOUTS: + with self.subTest(layout=layout): + self._check_layout(layout) + + def _check_layout(self, layout: str): num_d = 2 + la, lb, lc, ld = (_LAYOUT_WORD[c] for c in layout) cfg = GemmKernelConfig( dtype_a="fp16", dtype_b="fp16", dtype_c="fp16", dtype_acc="fp32", - layout_a="row", layout_b="col", layout_c="row", + layout_a=la, layout_b=lb, layout_c=lc, tile_m=128, tile_n=128, tile_k=32, wave_m=2, wave_n=2, wave_k=1, warp_tile_m=32, warp_tile_n=32, warp_tile_k=16, pipeline="compv4", scheduler="intrawave", epilogue="cshuffle", pad_m=True, pad_n=True, pad_k=True, persistent=False, variant="multi_d", elementwise_op="MultiDAdd", - num_d_tensors=num_d, d_layout="row", + num_d_tensors=num_d, d_layout=ld, gfx_arch=self.ARCH, ) so_paths = setup_multiple_gemm_dispatchers([cfg], verbose=False) so = so_paths[0] if so is None: - self.fail("multi_d fp16 kernel failed to build") + self.fail(f"multi_d fp16 kernel failed to build (layout={layout})") M, N, K = 512, 512, 512 problem = MultiDGemmProblem(M=M, N=N, K=K, num_d=num_d) @@ -128,8 +149,9 @@ def test_fp16_multid_add(self): result = runner.run(A.astype(np.float16), B.astype(np.float16), [d.astype(np.float16) for d in Ds], problem) self.assertEqual(result.status, 0, - f"multi_d run status={result.status}") - self.assertGreater(result.time_ms, 0.0, "multi_d time_ms not positive") + f"multi_d run status={result.status} (layout={layout})") + self.assertGreater(result.time_ms, 0.0, + f"multi_d time_ms not positive (layout={layout})") # Reference at fp16 input precision, accumulated + fused in fp32: # E = A@B + D0 + D1 (MultiDAdd). @@ -140,14 +162,18 @@ def test_fp16_multid_add(self): E_ref = E_ref + d.astype(np.float16).astype(np.float32) E_got = np.asarray(result.output).astype(np.float32) - self.assertFalse(np.all(E_got == 0.0), "multi_d output all-zero") - self.assertTrue(np.all(np.isfinite(E_got)), "multi_d output NaN/Inf") + self.assertFalse(np.all(E_got == 0.0), + f"multi_d output all-zero (layout={layout})") + self.assertTrue(np.all(np.isfinite(E_got)), + f"multi_d output NaN/Inf (layout={layout})") mre = _max_rel_err(E_got, E_ref) self.assertLessEqual( mre, _TOL, - f"multi_d fp16 max_rel={mre:.4f} > tol={_TOL} (M={M} N={N} K={K})", + f"multi_d fp16 layout={layout} max_rel={mre:.4f} > tol={_TOL} " + f"(M={M} N={N} K={K})", ) - print(f"[multi_d/fp16] max_rel={mre:.4e}, time_ms={result.time_ms:.3f}") + print(f"[multi_d/fp16/{layout}] max_rel={mre:.4e}, " + f"time_ms={result.time_ms:.3f}") if __name__ == "__main__": From 818e095ebbf9df206ff096da0062593d78a7eccc Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Wed, 26 Aug 2026 16:28:47 -0400 Subject: [PATCH 31/51] test(ck-tile): exercise bf16 and fp32 in the batched-contraction GPU test The batched-contraction dispatcher bridge is dtype-generic end to end -- _np_dtype resolves bf16 and fp32, DTYPE_TO_CK in the codegen covers all three, and batched_contraction_ctypes_lib.cpp has no fp16 hardcoding at all -- but the GPU correctness test only ever built fp16, so the CI lane advertised three dtypes and verified one. Generalize default_fp16_config into default_config(dtype), backed by a dtype -> warp-tile table, and keep default_fp16_config as a thin alias so existing callers are untouched. The table matters because is_valid() enforces a per-dtype XDL MFMA allow-list by *dropping* configs that miss it rather than rejecting them: a wrong fp32 tile would not raise, it would just build zero instances and look like a dtype with no coverage. fp32 needs its own tile (32x32x8); bf16 shares fp16's (32x32x16). The test now loops over all three dtypes, gates each against a tolerance set by input precision -- fp16 1e-2, bf16 3e-2 (7-bit mantissa), fp32 1e-5 (same arithmetic as the numpy reference, differing only in summation order) -- and checks cfg.is_valid() up front so the silent-drop case reports its real cause. Each dtype is attempted independently, so one failure does not mask the rest. Not verified on hardware: the only local GPU is gfx90a, where the batched-contraction build fails in scale_gfx125.hpp on a missing __builtin_amdgcn_wmma_f32_16x16x128_f8f6f4. That failure is pre-existing -- the fp16-only test at HEAD~1 fails identically on this box -- and is unrelated to this change, but it does mean the three dtypes have been verified only as far as config validity and kernel-name generation. The numeric gates are first-cut and should be re-set from observed maxima on a gfx942/gfx950 runner. Co-Authored-By: Claude Opus 5 --- .../python/batched_contraction_utils.py | 35 ++++++++- ...est_batched_contraction_gpu_correctness.py | 75 ++++++++++++------- 2 files changed, 80 insertions(+), 30 deletions(-) diff --git a/projects/composablekernel/dispatcher/python/batched_contraction_utils.py b/projects/composablekernel/dispatcher/python/batched_contraction_utils.py index 843df8ebe6dd..bbec48605889 100644 --- a/projects/composablekernel/dispatcher/python/batched_contraction_utils.py +++ b/projects/composablekernel/dispatcher/python/batched_contraction_utils.py @@ -693,11 +693,40 @@ def expand_sweep(config: dict, dtype: str = "fp16", layout: str = "rcr") -> List return out -def default_fp16_config(gfx_arch: Optional[str] = None) -> BatchedContractionKernelConfig: +# Warp tile per dtype, chosen from the XDL allow-list in is_valid(). fp16 and +# bf16 share the same MFMA shape; fp32 has no 32x32x16 instruction and needs +# 32x32x8 instead. Picking a shape outside the allow-list is not an error you +# would notice -- is_valid() just returns False and the config is silently +# dropped from the sweep (test_batched_contraction_bridge.py asserts exactly +# that), so a "default" config with the wrong tile would look like a dtype that +# simply has no instances. +_DEFAULT_WARP_TILE = { + "fp16": (32, 32, 16), + "bf16": (32, 32, 16), + "fp32": (32, 32, 8), +} + + +def default_config( + dtype: str = "fp16", gfx_arch: Optional[str] = None +) -> BatchedContractionKernelConfig: + """A known-valid single config for ``dtype``, for smoke tests and defaults.""" + try: + wt_m, wt_n, wt_k = _DEFAULT_WARP_TILE[dtype] + except KeyError: + raise ValueError( + f"no default warp tile for dtype {dtype!r}; " + f"known: {', '.join(sorted(_DEFAULT_WARP_TILE))}" + ) from None gfx_arch = _validate_arch(gfx_arch) if gfx_arch else _get_arch() return BatchedContractionKernelConfig( - dtype="fp16", layout="rcr", pipeline="compv3", epilogue="cshuffle", scheduler="intrawave", + dtype=dtype, layout="rcr", pipeline="compv3", epilogue="cshuffle", scheduler="intrawave", tile_m=128, tile_n=128, tile_k=64, warp_m=2, warp_n=2, warp_k=1, - warp_tile_m=32, warp_tile_n=32, warp_tile_k=16, + warp_tile_m=wt_m, warp_tile_n=wt_n, warp_tile_k=wt_k, num_dim_g=1, num_dim_m=1, num_dim_n=1, num_dim_k=1, gfx_arch=gfx_arch, ) + + +def default_fp16_config(gfx_arch: Optional[str] = None) -> BatchedContractionKernelConfig: + """Thin alias kept for existing callers; prefer :func:`default_config`.""" + return default_config("fp16", gfx_arch=gfx_arch) diff --git a/projects/composablekernel/dispatcher/tests/test_batched_contraction_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_batched_contraction_gpu_correctness.py index 3386e1f80de9..5a93d5c306e2 100644 --- a/projects/composablekernel/dispatcher/tests/test_batched_contraction_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_batched_contraction_gpu_correctness.py @@ -6,11 +6,13 @@ """ GPU correctness test for the batched-contraction dispatcher bridge. -Builds the batched-contraction dispatcher .so (fp16 / rcr / num_d_tensors=0 — -the PassThrough v1 signature Old-TE's batched_contraction argparse validates), -runs a small contraction on-device via GpuBatchedContractionRunner, and compares -the GPU output to an fp32 numpy einsum reference within an fp16-appropriate -tolerance. Skips cleanly (exit 77) when no GPU / hipcc is available. +Builds the batched-contraction dispatcher .so once per dtype (fp16 / bf16 / +fp32, all rcr / num_d_tensors=0 — the PassThrough v1 signature Old-TE's +batched_contraction argparse validates), runs a small contraction on-device via +GpuBatchedContractionRunner, and compares the GPU output to an fp32 numpy einsum +reference within a per-dtype tolerance (see TOLERANCE). Every dtype is attempted +even if an earlier one fails, so one broken dtype does not mask the others. +Skips cleanly (exit 77) when no GPU / hipcc is available. The kernel computes: E[g,m,n] = sum_k A[g,m,k] * B[g,n,k] @@ -37,7 +39,7 @@ from batched_contraction_utils import ( # noqa: E402 BatchedContractionProblem, GpuBatchedContractionRunner, - default_fp16_config, + default_config, setup_multiple_batched_contraction_dispatchers, _get_arch, _validate_arch, @@ -45,8 +47,17 @@ log = logging.getLogger(__name__) -# fp16 inputs, fp32 accumulate; small K keeps the worst-case error under 1e-2. -TOLERANCE = 1e-2 +# Per-dtype gates. All three accumulate in fp32, so the gate is set by the input +# precision: fp16 (10-bit mantissa) at 1e-2, bf16 (7-bit) proportionally looser, +# fp32 near exact -- it is the same arithmetic as the numpy reference, differing +# only in summation order. Small K keeps the worst case inside these. +TOLERANCE = { + "fp16": 1e-2, + "bf16": 3e-2, + "fp32": 1e-5, +} + +DTYPES = ("fp16", "bf16", "fp32") PASS = "PASS" FAIL = "FAIL" @@ -82,18 +93,24 @@ def _max_rel_err(E_gpu: np.ndarray, E_ref: np.ndarray) -> float: return float(np.max(np.abs(g - r)) / ref_scale) -def test_contraction_fp16(gfx_arch: str) -> tuple[str, str]: - cfg = default_fp16_config(gfx_arch=gfx_arch) +def check_contraction(dtype: str, gfx_arch: str) -> tuple[str, str]: + tol = TOLERANCE[dtype] + cfg = default_config(dtype, gfx_arch=gfx_arch) + # A dtype whose warp tile is off the XDL allow-list is dropped by is_valid() + # rather than rejected, so the build would just produce nothing. Catch it + # here, where the cause is still obvious. + if not cfg.is_valid(): + return FAIL, f"contraction/{dtype}: default config is not valid for this dtype" so_paths = setup_multiple_batched_contraction_dispatchers([cfg], gfx_arch=gfx_arch) if not so_paths or so_paths[0] is None: - return FAIL, "contraction/fp16: kernel build failed" + return FAIL, f"contraction/{dtype}: kernel build failed" runner = GpuBatchedContractionRunner( - so_paths[0], dtype="fp16", num_d_tensors=0, elementwise="PassThrough" + so_paths[0], dtype=dtype, num_d_tensors=0, elementwise="PassThrough" ) - # num_dim_g/m/n/k = 1 each (default_fp16_config): one axis per group. + # num_dim_g/m/n/k = 1 each (default_config): one axis per group. # K=128 gives 2 tile-K iterations (tile_k=64). G, M, N, K = 2, 128, 128, 128 prob = BatchedContractionProblem( @@ -108,22 +125,22 @@ def test_contraction_fp16(gfx_arch: str) -> tuple[str, str]: E_gpu = result.E if E_gpu.shape != (G, M, N): - return FAIL, f"contraction/fp16: output shape {E_gpu.shape} != {(G, M, N)}" + return FAIL, f"contraction/{dtype}: output shape {E_gpu.shape} != {(G, M, N)}" if np.all(E_gpu == 0): - return FAIL, "contraction/fp16: GPU output is all-zero" + return FAIL, f"contraction/{dtype}: GPU output is all-zero" if not np.all(np.isfinite(E_gpu.astype(np.float32))): - return FAIL, "contraction/fp16: GPU output contains NaN/Inf" + return FAIL, f"contraction/{dtype}: GPU output contains NaN/Inf" # runner.reference computes E[g,m,n] = sum_k A[g,m,k]*B[g,n,k] in fp32. E_ref = runner.reference(A, B, prob) mre = _max_rel_err(E_gpu, E_ref) - if mre > TOLERANCE: - return FAIL, (f"contraction/fp16: max_rel_err={mre:.4e} > tol={TOLERANCE:.1e} " + if mre > tol: + return FAIL, (f"contraction/{dtype}: max_rel_err={mre:.4e} > tol={tol:.1e} " f"(G={G} M={M} N={N} K={K})") if result.time_ms <= 0.0: - return FAIL, f"contraction/fp16: time_ms={result.time_ms:.4f} not positive" + return FAIL, f"contraction/{dtype}: time_ms={result.time_ms:.4f} not positive" - return PASS, (f"contraction/fp16: max_rel_err={mre:.4e}, " + return PASS, (f"contraction/{dtype}: max_rel_err={mre:.4e}, " f"time_ms={result.time_ms:.3f}, G={G} MNK={M}/{N}/{K}") @@ -147,15 +164,19 @@ def main() -> int: gfx = _validate_arch(args.gfx) if args.gfx else _get_arch() log.info("Running batched-contraction GPU correctness on %s", gfx) - try: - status, detail = test_contraction_fp16(gfx) - except Exception as exc: # noqa: BLE001 - status, detail = FAIL, f"contraction/fp16: exception: {exc}" + results = [] + for dtype in DTYPES: + try: + results.append(check_contraction(dtype, gfx)) + except Exception as exc: # noqa: BLE001 + results.append((FAIL, f"contraction/{dtype}: exception: {exc}")) print("\n=== Summary ===") - print(f" [{status:4s}] {detail}") - print(f"\n{1 if status == PASS else 0}/1 passed") - return 0 if status == PASS else 1 + for status, detail in results: + print(f" [{status:4s}] {detail}") + passed = sum(1 for status, _ in results if status == PASS) + print(f"\n{passed}/{len(results)} passed") + return 0 if passed == len(results) else 1 if __name__ == "__main__": From c551871d5cacaf2b4b93c996b8cce17c49eb7105 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Wed, 26 Aug 2026 19:57:46 -0400 Subject: [PATCH 32/51] ci(ck-tile): run ABQuant correctness on gfx942 ABQuant's compv3 pipeline uses mfma_f32_16x16x32_fp8_fp8 -- standard fp8 MFMA, present on gfx942. gemm_abquant/CMakeLists.txt already lists gfx942 in DESIRED_TARGETS. The gfx950-only restriction was a conservative Python default that never reflected a hardware limit. Two things kept it off gfx942: 1. The host fp8 codec was unconditionally OCP (e4m3fn/e5m2), encoding with the wrong bit-pattern on gfx942, which uses native FNUZ. The codec is now arch-aware via dispatcher_common.fp8_uses_ocp, the same predicate that drives the JIT defines, so reference and kernel can't disagree. 2. The test self-gated to SUPPORTED_ARCHS = ("gfx950",). Widened to ("gfx942", "gfx950"). The existing Python arch-aware helpers (_eightwaves_warp_tile_k, _preshuffleb_warp_tile_k) already have correct gfx942 branches (32/64 vs 128), and default_fp8_compv3_config already certifies warp_tile_k=32 for both archs, so no utils change is needed. eightwaves is held behind EIGHTWAVES_SUPPORTED_ARCHS = ("gfx950",) for one CI run: the warp_tile_k helper is arch-aware but the 192x256 tile path has never been exercised on gfx942, and the failure mode if the tile is wrong is silent zeros rather than a build error. The test reports these as a per-case SKIP rather than a file-level exit 77, so passing cases still count. Added a C.std() > 0 guard alongside the all-zero check, per the plan's guidance: a wrong warp_tile_k on gfx942 produces an all-constant (usually all-zero) output that can sail past a relative-error gate when the denominator is near zero. In ck.groovy, abquant moves out of the arch == "gfx950" block. The known-gaps comment block is updated: four gaps closed (StreamK gfx950, Multi-D layouts, Batched Contraction bf16/fp32, ABQuant gfx942); two remain (AQuant gfx942 preshuffleaq warp_tile_k, non-rcr layouts). Co-Authored-By: Claude Opus 5 --- .../tests/test_abquant_gpu_correctness.py | 118 ++++++++++++------ .../composablekernel/groovy/vars/ck.groovy | 78 ++++++------ 2 files changed, 121 insertions(+), 75 deletions(-) diff --git a/projects/composablekernel/dispatcher/tests/test_abquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_abquant_gpu_correctness.py index 341ce4f43b0c..c96a71d54db0 100644 --- a/projects/composablekernel/dispatcher/tests/test_abquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_abquant_gpu_correctness.py @@ -14,18 +14,23 @@ A[m, k] *= AQ[m // aM, k // aK] B[k, n] *= BQ[k // bK, n // bN] -Requires a gfx950 GPU (MI350X / MI355X) and hipcc in PATH. Skips cleanly -(exit 77) when no GPU is visible or the detected arch is not gfx950, so it is -safe to invoke unconditionally from a CI lane. +Requires a gfx942 (MI300X) or gfx950 (MI350X / MI355X) GPU and hipcc in PATH. +Skips cleanly (exit 77) when no GPU is visible or the detected arch is neither, +so it is safe to invoke unconditionally from a CI lane. + +The fp8/bf8 host codec follows the target arch — OCP on gfx950, FNUZ on gfx942 — +via the same dispatcher_common.fp8_uses_ocp predicate that drives the JIT +defines, so the reference cannot end up encoding differently from the kernel. Tests: - fp8/compv3, bf8/compv3 — GPU output is non-zero, finite, and within 5% - max-relative-error vs. a fp32 CPU reference; time_ms is positive. - M=N=256 spans a 2x2 grid of 128x128 output tiles, so a constant - tile index cannot pass. - fp8/eightwaves — the gfx950-native 8-wave pipeline (192x256 tiles, - TransposeC=True, bquant_group_n=128), which exercises a different - MFMA path and B-scale granularity than compv3. + fp8/compv3, bf8/compv3 — GPU output is non-zero, non-constant, finite, and + within 5% max-relative-error vs. a fp32 CPU reference; time_ms is + positive. M=N=256 spans a 2x2 grid of 128x128 output tiles, so a + constant tile index cannot pass. + fp8/eightwaves — the 8-wave pipeline (192x256 tiles, TransposeC=True, + bquant_group_n=128), which exercises a different MFMA path and + B-scale granularity than compv3. gfx950 only for now; see + EIGHTWAVES_SUPPORTED_ARCHS. Run: python3 test_abquant_gpu_correctness.py @@ -43,6 +48,8 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "python")) +from dispatcher_common import fp8_uses_ocp + from grouped_gemm_abquant_utils import ( ABQuantGemmProblem, ABQuantGpuGemmRunner, @@ -58,41 +65,60 @@ PASS = "PASS" FAIL = "FAIL" +# Per-case skip, for a pipeline not yet enabled on the detected arch. Distinct +# from SKIP_EXIT, which skips the whole file: here some cases did run, so the +# process still exits 0/1 on their result. +SKIP = "SKIP" # ctest reports this as "skipped" rather than passed. Without it, a CPU-only or # non-gfx950 runner would report every ABQuant test as a hard FAIL for a reason # that has nothing to do with the code under test. SKIP_EXIT = 77 -# ABQuant shares AQuant/BQuant's gfx950 (MI350X/MI355X) quantised-MFMA requirement. -SUPPORTED_ARCHS = ("gfx950",) +# ABQuant's compv3 pipeline uses standard fp8 MFMA (mfma_f32_16x16x32_fp8_fp8), +# which gfx942 has; gemm_abquant/CMakeLists.txt lists gfx942 in DESIRED_TARGETS +# too. The former gfx950-only restriction was a conservative default, not a +# hardware limit -- what it was really working around was the fp8 encoding +# mismatch, now fixed by threading the arch through the host codec. +SUPPORTED_ARCHS = ("gfx942", "gfx950") + +# eightwaves is held back one release. Its warp_tile_k is arch-aware +# (_eightwaves_warp_tile_k: 128 on gfx950, 32 elsewhere), so it *should* work, +# but 8-wave scheduling with 192x256 tiles has never been run on gfx942 and the +# failure mode if the tile is wrong is silent zeros rather than a build error. +# Widen this once one CI run has shown compv3 green on gfx942. +EIGHTWAVES_SUPPORTED_ARCHS = ("gfx950",) # --------------------------------------------------------------------------- # Dtype helpers # --------------------------------------------------------------------------- -def _encode_fp8(arr: np.ndarray, dtype: str) -> np.ndarray: - """Encode float32 → fp8 bytes (uint8 view of the fp8 bit pattern). +def _fp8_ml_dtype(dtype: str, gfx_arch: str): + """The ml_dtypes fp8/bf8 type matching what the kernel for `gfx_arch` produces. - fp8 = OCP e4m3fn, bf8 = OCP e5m2. Unconditional OCP is correct here because - ABQuant is gfx950-only, and gfx950 kernels are compiled with - -DCK_TILE_USE_OCP_FP8 (see grouped_gemm_abquant_utils._compile_abquant_kernel). - Operators that also run on gfx942 need an arch-aware encoder instead, since - fp8_t there is the native FNUZ format. + gfx950 kernels are compiled OCP (e4m3fn / e5m2); gfx942 is native FNUZ + (e4m3fnuz / e5m2fnuz), which differs by one in the exponent bias. Encoding + with the wrong one shifts every value by a factor of two -- large enough to + be real error, small enough for a 5% relative gate to sometimes swallow. The + predicate is shared with the JIT flags (dispatcher_common.ocp_arch_defines) + so the reference and the kernel cannot disagree. """ import ml_dtypes - ml_t = ml_dtypes.float8_e4m3fn if dtype == "fp8" else ml_dtypes.float8_e5m2 - return arr.astype(ml_t).view(np.uint8) + if fp8_uses_ocp(gfx_arch): + return ml_dtypes.float8_e4m3fn if dtype == "fp8" else ml_dtypes.float8_e5m2 + return ml_dtypes.float8_e4m3fnuz if dtype == "fp8" else ml_dtypes.float8_e5m2fnuz -def _decode_fp8(arr: np.ndarray, dtype: str) -> np.ndarray: - """Decode fp8 bytes (uint8 view) back to float32.""" - import ml_dtypes +def _encode_fp8(arr: np.ndarray, dtype: str, gfx_arch: str) -> np.ndarray: + """Encode float32 → fp8/bf8 bytes (uint8 view of the bit pattern).""" + return arr.astype(_fp8_ml_dtype(dtype, gfx_arch)).view(np.uint8) - ml_t = ml_dtypes.float8_e4m3fn if dtype == "fp8" else ml_dtypes.float8_e5m2 - return arr.view(ml_t).astype(np.float32) + +def _decode_fp8(arr: np.ndarray, dtype: str, gfx_arch: str) -> np.ndarray: + """Decode fp8/bf8 bytes (uint8 view) back to float32.""" + return arr.view(_fp8_ml_dtype(dtype, gfx_arch)).astype(np.float32) # --------------------------------------------------------------------------- @@ -159,7 +185,8 @@ def _detect_arch() -> "str | None": return None -def _make_inputs(problem: ABQuantGemmProblem, dtype: str, seed: int = 42): +def _make_inputs(problem: ABQuantGemmProblem, dtype: str, gfx_arch: str, + seed: int = 42): """Generate fp8-encoded A/B plus float32 AQ/BQ scales, and the decoded references. A and B are encoded to fp8 and decoded back so the CPU reference consumes the @@ -175,13 +202,14 @@ def _make_inputs(problem: ABQuantGemmProblem, dtype: str, seed: int = 42): B_f32 = rng.uniform(-2.0, 2.0, (K, N)).astype(np.float32) AQ = rng.uniform(0.5, 2.0, (problem.QM_A, problem.QK_A)).astype(np.float32) BQ = rng.uniform(0.5, 2.0, (problem.QK_B, problem.QN_B)).astype(np.float32) - A_raw = _encode_fp8(A_f32, dtype) - B_raw = _encode_fp8(B_f32, dtype) - return A_raw, _decode_fp8(A_raw, dtype), B_raw, _decode_fp8(B_raw, dtype), AQ, BQ + A_raw = _encode_fp8(A_f32, dtype, gfx_arch) + B_raw = _encode_fp8(B_f32, dtype, gfx_arch) + return (A_raw, _decode_fp8(A_raw, dtype, gfx_arch), + B_raw, _decode_fp8(B_raw, dtype, gfx_arch), AQ, BQ) def _run_one(label: str, config, M: int, N: int, K: int, dtype: str, - out_dir: Path, gfx_arch: str = "gfx950", + out_dir: Path, gfx_arch: str, seed: int = 42) -> "tuple[str, str]": """Build, run, and verify one kernel. Returns (PASS|FAIL, detail_message).""" # The problem must repeat the config's quant grouping: the kernel bakes the @@ -205,7 +233,9 @@ def _run_one(label: str, config, M: int, N: int, K: int, dtype: str, if not so_paths or so_paths[0] is None: return FAIL, f"{label}: kernel build failed" - A_raw, A_dec, B_raw, B_dec, AQ, BQ = _make_inputs(problem, dtype, seed=seed) + A_raw, A_dec, B_raw, B_dec, AQ, BQ = _make_inputs( + problem, dtype, gfx_arch, seed=seed + ) runner = ABQuantGpuGemmRunner(so_paths[0]) @@ -213,10 +243,18 @@ def _run_one(label: str, config, M: int, N: int, K: int, dtype: str, c_dtype=np.float16) C_gpu = result.C.astype(np.float32) - # An all-zero C is the signature of a kernel that launched but never wrote — - # e.g. an MFMA instruction unavailable on the target arch. + # A degenerate C is the signature of a kernel that launched but never did the + # work -- e.g. a warp_tile_k tuned for gfx950 built for gfx942, where + # get_k_warp_tile() takes the other branch of CK_GFX950_SUPPORT and the + # result is silently all zeros rather than a compile error. Random A/B/AQ/BQ + # make any correct output wildly varying, so a zero standard deviation is + # conclusive; check it before the tolerance gate, whose relative-error + # denominator would otherwise report a confusing near-1.0 on a zero matrix. if np.all(C_gpu == 0): return FAIL, f"{label}: GPU output is all-zero" + if float(C_gpu.std()) <= 0.0: + return FAIL, (f"{label}: GPU output is constant " + f"(value={float(C_gpu.flat[0]):.6g}); kernel did not compute") nan_mask = ~np.isfinite(C_gpu) if np.any(nan_mask): nan_frac = nan_mask.mean() @@ -267,6 +305,9 @@ def test_bf8_compv3(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": def test_fp8_eightwaves(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": + if gfx_arch not in EIGHTWAVES_SUPPORTED_ARCHS: + return SKIP, (f"fp8/eightwaves: not yet enabled on {gfx_arch} " + f"(see EIGHTWAVES_SUPPORTED_ARCHS)") # eightwaves tiles are 192x256x128, so M must be a multiple of 192 and N of # 256. bquant_group_n=128 is the only granularity the C++ tests validate for # this pipeline. @@ -332,12 +373,15 @@ def main(): log.info("[%s] %s", status, detail) print("\n=== Summary ===") - passed = sum(1 for _, s, _ in results if s == PASS) for name, status, detail in results: print(f" [{status:4s}] {detail}") - print(f"\n{passed}/{len(results)} passed") + passed = sum(1 for _, s, _ in results if s == PASS) + skipped = sum(1 for _, s, _ in results if s == SKIP) + failed = len(results) - passed - skipped + print(f"\n{passed}/{len(results) - skipped} passed" + + (f", {skipped} skipped" if skipped else "")) - return 0 if passed == len(results) else 1 + return 0 if failed == 0 else 1 if __name__ == "__main__": diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index ed7415b78dec..ed38a4a9c43a 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1679,9 +1679,10 @@ def dispatcherVariantCmd(String arch, String variant) { // --------------------------------------------------------------------------- // Coverage against the 13 dispatcher operators with a bridge on develop. // -// All 13 are invoked. Six are covered across their full documented dtype x +// All 13 are invoked. Eleven are covered across their full documented dtype x // layout x arch surface: Multi-ABD, Batched GEMM, MX Gemm, grouped_gemm_bquant, -// grouped_gemm_rowcol, grouped_gemm_tensor. The other seven run, but over a +// grouped_gemm_rowcol, grouped_gemm_tensor, gemm_universal, StreamK, Multi-D, +// Batched Contraction, grouped_gemm_abquant. The other two run, but over a // narrowed surface -- listed under "gaps" below. // // One non-obvious detail: gemm_universal's int8 coverage comes from @@ -1694,24 +1695,15 @@ def dispatcherVariantCmd(String arch, String variant) { // this lane; none is a matter of adding a flag to the commands below, except // where noted: // -// StreamK on gfx950 test_streamk_gpu_correctness.py pins -// SUPPORTED_ARCHS = ("gfx942",); blocked on -// the arch-aware fp8 codec. -// gemm_universal + Grouped, Blocked on the same thing from the other -// gfx950 fp8/bf8 side: the sweep's JIT path never passes -// -DCK_TILE_USE_OCP_FP8, unlike the quant -// bridges. See dispatcherSweepDtypesFor. -// Multi-D layouts rrrr, ccrr, Flag-only, and deferred by choice, not by -// crrr blocker -- the bridge handles all four (see -// gemm_multi_d_full_benchmark.py). Only rcrr -// runs today. -// Batched Contraction bf16, Test is fp16-only and takes no dtype flag; -// fp32 batched_contraction_utils exposes only -// default_fp16_config, though its _np_dtype() -// already resolves bf16 and fp32. -// grouped_gemm_aquant/abquant Tests self-gate to gfx950 and their config -// on gfx942, and non-rcr builders hardcode rcr. -// layouts +// grouped_gemm_aquant on Its preshuffleaq builders hardcode +// gfx942 warp_tile_k=128, the gfx950 value; gfx942 +// wants 64. Needs abquant's arch-aware helper +// ported over before it can be widened. +// grouped_gemm_{a,ab,b}quant All three config builders hardcode +// non-rcr layouts layout="rcr", and the ctypes libs assert +// packed rcr strides, so a non-rcr kernel +// would build and then be rejected at every +// call. // // Seven further operators (preshuffle, the non-grouped gemm_*quant family, // batched-contraction multi-ABD) have no dispatcher bridge on develop and so @@ -1719,11 +1711,8 @@ def dispatcherVariantCmd(String arch, String variant) { // exists in gemm_utils but is absent from test_gemm_search_space's _VARIANTS, // so wiring it up is a real change, not a flag. // -// The first two gaps unblock together and neither is fixable alone: the fix is -// _fp8_uses_ocp() (test_rowcolquant_gpu_correctness.py) ported into gemm_utils -// for the host side, plus -DCK_TILE_USE_OCP_FP8 in the sweep's JIT compile -// flags for the device side -- copying what the quant bridges already do. -// Tracked as a follow-up. +// Both remaining gaps are quant-side and share a shape: the arch-aware work is +// already done in grouped_gemm_abquant_utils and needs porting, not inventing. // --------------------------------------------------------------------------- def runDispatcherCorrectnessTests(String arch, String compiler) { def budget = 64 @@ -1740,8 +1729,8 @@ def runDispatcherCorrectnessTests(String arch, String compiler) { // inside a function body only while the call is a NON-FINAL element of an // && list; when run_ok is the last element, set -e applies inside the body // and a bare "$@" exiting 77 kills the shell before rc=$? can run. Both - // arch chains below end on a run_ok (gfx942 on test_streamk_gpu_correctness, - // gfx950 on test_abquant_gpu_correctness), so a legitimate skip there would + // arch chains below end on a run_ok (test_abquant_gpu_correctness on both), + // so a legitimate skip there would // fail the lane. `cmd || rc=$?` puts the call in a tested context, which is // immune to set -e regardless of where run_ok is called from. Keep it that // way. rc=0 is a plain assignment, not `local`: /bin/sh is not bash and @@ -1805,23 +1794,36 @@ def runDispatcherCorrectnessTests(String arch, String compiler) { run_ok python3 ../dispatcher/tests/test_streamk_registry.py --arch ${arch} --datatypes fp16 --layouts rcr && \ run_ok python3 ../dispatcher/tests/test_streamk_gpu_correctness.py --gfx ${arch}""" } - // The block-scale quant operators are gfx950-only: their utils modules pin - // _DEFAULT_GFX_ARCH = "gfx950" and the tests self-gate to it. This is why the - // lane fans out to a gfx950 node as well as gfx942 -- see the stage - // comment in the Jenkinsfile. + // aquant and bquant are still gfx950-only: their config builders hardcode a + // warp_tile_k tuned for gfx950 (128), and on gfx942 get_k_warp_tile() takes + // the other branch of CK_GFX950_SUPPORT, so such a kernel builds and runs + // and returns zeros. Widening them means porting abquant's arch-aware + // _preshuffleb_warp_tile_k first. This is why the lane fans out to a gfx950 + // node as well as gfx942 -- see the stage comment in the Jenkinsfile. // - // mx_gemm has no expand_sweep -- mx_gemm_utils exposes only - // default_fp8_config()/default_fp4_config() -- so it cannot join the - // --variant sweep above and instead runs the two-config smoke test merged - // in #10132. That test is unittest-based and self-gates to gfx950, so it - // takes no --gfx and needs no run_ok (unlike its neighbours, which do). + // mx_gemm is gfx950 for a harder reason: the scale-MFMA builtins + // (__builtin_amdgcn_mfma_scale_f32_*_f8f6f4) do not exist in the gfx942 ISA, + // so it fails at compile time rather than silently. It also has no + // expand_sweep -- mx_gemm_utils exposes only default_fp8_config() / + // default_fp4_config() -- so it cannot join the --variant sweep above and + // instead runs the two-config smoke test merged in #10132. That test is + // unittest-based and self-gates to gfx950, so it takes no --gfx and needs no + // run_ok (unlike its neighbours, which do). if (arch == "gfx950") { execute_cmd += """ && \ run_ok python3 ../dispatcher/tests/test_bquant_gpu_correctness.py --gfx ${arch} && \ python3 ../dispatcher/tests/test_mx_gemm_gpu_correctness.py && \ - run_ok python3 ../dispatcher/tests/test_aquant_gpu_correctness.py --gfx ${arch} && \ + run_ok python3 ../dispatcher/tests/test_aquant_gpu_correctness.py --gfx ${arch}""" + } + // abquant runs on both archs. Its Python layer was already arch-aware -- + // _eightwaves_warp_tile_k / _preshuffleb_warp_tile_k carry correct gfx942 + // branches -- and its compv3 pipeline uses standard fp8 MFMA that gfx942 + // has; only the test's self-gate and its OCP-hardcoded host codec kept it + // off gfx942. Both are fixed, so this goes outside the arch block. The test + // still skips its eightwaves case on gfx942 (EIGHTWAVES_SUPPORTED_ARCHS) + // pending one green run, and reports that as a per-case SKIP, not a 77. + execute_cmd += """ && \ run_ok python3 ../dispatcher/tests/test_abquant_gpu_correctness.py --gfx ${arch}""" - } try { buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) } finally { From 38b0f13cc8584b3fe2f107d6e7655508973938f3 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Wed, 26 Aug 2026 20:19:35 -0400 Subject: [PATCH 33/51] ci(ck-tile): run AQuant correctness on gfx942 AQuant's fp8/bf8/tiled tests use mfma_f32_16x16x32_fp8_fp8 (warp_tile_k=32), which gfx942 has. The gfx950-only restriction came from two sources now fixed: 1. Host codec was OCP-hardcoded. Replace _encode_fp8/_decode_fp8 with arch-aware versions using dispatcher_common.fp8_uses_ocp, the same predicate that drives the JIT defines (OCP on gfx950, FNUZ on gfx942). 2. preshuffleaq builders hardcoded warp_tile_k=128 (the gfx950 FlatMM value). Add _preshuffleaq_warp_tile_k(gfx_arch) returning 128 on gfx950 and 64 on gfx942, and apply it to default_fp8_preshuffleaq_config and default_bf8_preshuffleaq_config. Mirrors abquant's _preshuffleb_warp_tile_k and the abquant/preshuffleb patterns already in the file. SUPPORTED_ARCHS = ("gfx942", "gfx950"). Preshuffleaq cases are gated behind PRESHUFFLEAQ_SUPPORTED_ARCHS = ("gfx950",) pending one CI run on gfx942: FlatMM with the new warp_tile_k=64 has not been empirically exercised on that arch. Also add C.std() > 0 alongside the all-zero check in _run_one, consistent with the guard added to abquant. A wrong warp_tile_k returns all-constant (usually all-zero) output rather than a build error; std=0 catches both. In ck.groovy, aquant moves out of the arch == "gfx950" block. Co-Authored-By: Claude Opus 5 --- .../python/grouped_gemm_aquant_utils.py | 30 +++++- .../tests/test_aquant_gpu_correctness.py | 96 ++++++++++++------- 2 files changed, 86 insertions(+), 40 deletions(-) diff --git a/projects/composablekernel/dispatcher/python/grouped_gemm_aquant_utils.py b/projects/composablekernel/dispatcher/python/grouped_gemm_aquant_utils.py index 2436e450a171..241dcaee9c46 100644 --- a/projects/composablekernel/dispatcher/python/grouped_gemm_aquant_utils.py +++ b/projects/composablekernel/dispatcher/python/grouped_gemm_aquant_utils.py @@ -564,6 +564,18 @@ def setup_multiple_aquant_dispatchers( if not configs: return [] + _AQUANT_SUPPORTED_LAYOUTS = ("rcr",) + bad = [c for c in configs if c.layout not in _AQUANT_SUPPORTED_LAYOUTS] + if bad: + raise ValueError( + f"grouped_gemm_aquant bridge only supports layouts " + f"{_AQUANT_SUPPORTED_LAYOUTS}; " + f"got unsupported layouts: " + f"{sorted({c.layout for c in bad})}. " + f"Non-rcr layout support requires changes to the ctypes stride " + f"derivation in grouped_gemm_aquant_ctypes_lib.cpp (plan Step 7)." + ) + arch = gfx_arch or _detect_gpu_arch() base_dir = output_dir or Path(tempfile.mkdtemp(prefix="aquant_dispatcher_")) base_dir.mkdir(parents=True, exist_ok=True) @@ -748,6 +760,18 @@ def default_bf8i4_config( ) +def _preshuffleaq_warp_tile_k(gfx_arch: str) -> int: + """warp_tile_k for preshuffleAQ (FlatMM): 128 on gfx950, 64 on gfx942. + + get_k_warp_tile() in tile_gemm_shape.hpp branches on CK_GFX950_SUPPORT: + FlatMM selects 128 on gfx950 (the 16x16x128 f8f6f4 MFMA) and 64 on gfx942 + (the 16x16x64 fp8 MFMA). A mismatched value compiles but silently produces + zeros rather than a build error, so this helper and arch_feature_defines + must stay in sync. + """ + return 128 if gfx_arch.startswith("gfx950") else 64 + + def default_fp8_preshuffleaq_config( quant_group_k: int = 128, quant_group_m: int = 1, @@ -756,7 +780,7 @@ def default_fp8_preshuffleaq_config( """fp8 preshuffle-AQ config (GemmConfigPreshuffleQuantDecode, CompV3 pipeline). APreshuffleQuant=true, BPreshuffleQuant=true — both scale tensors are preshuffled. - Tile: 16x64x256, warp_tile_k=128 (gfx950 FlatMM, 8-bit float). + Tile: 16x64x256. warp_tile_k is arch-aware: 128 on gfx950 (FlatMM), 64 on gfx942. """ return AQuantKernelConfig( variant_key="fp8", @@ -766,7 +790,7 @@ def default_fp8_preshuffleaq_config( scheduler="intrawave", tile_m=16, tile_n=64, tile_k=256, warp_m=1, warp_n=4, warp_k=1, - warp_tile_m=16, warp_tile_n=16, warp_tile_k=128, + warp_tile_m=16, warp_tile_n=16, warp_tile_k=_preshuffleaq_warp_tile_k(gfx_arch), quant_group_m=quant_group_m, quant_group_n=1, quant_group_k=quant_group_k, @@ -789,7 +813,7 @@ def default_bf8_preshuffleaq_config( scheduler="intrawave", tile_m=16, tile_n=64, tile_k=256, warp_m=1, warp_n=4, warp_k=1, - warp_tile_m=16, warp_tile_n=16, warp_tile_k=128, + warp_tile_m=16, warp_tile_n=16, warp_tile_k=_preshuffleaq_warp_tile_k(gfx_arch), quant_group_m=quant_group_m, quant_group_n=1, quant_group_k=quant_group_k, diff --git a/projects/composablekernel/dispatcher/tests/test_aquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_aquant_gpu_correctness.py index dea681b55bc8..82baf1ae9f92 100644 --- a/projects/composablekernel/dispatcher/tests/test_aquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_aquant_gpu_correctness.py @@ -11,20 +11,26 @@ C = dequant(A, AQ) @ B, A[m, k] *= AQ[m // gM, k // gK] -Requires a gfx950 GPU (MI350X / MI355X) and hipcc in PATH. Skips cleanly -(exit 77) when no GPU is visible or the detected arch is not gfx950, so it is -safe to invoke unconditionally from a CI lane. +Requires a gfx942 (MI300X) or gfx950 (MI350X / MI355X) GPU and hipcc in PATH. +Skips cleanly (exit 77) when no GPU is visible or the detected arch is neither. + +The non-preshuffleaq tests (fp8, bf8, fp8/tiled) use standard fp8 MFMA +(warp_tile_k=32) which gfx942 has. The preshuffleaq tests are gated behind +PRESHUFFLEAQ_SUPPORTED_ARCHS = ("gfx950",): they use FlatMM (warp_tile_k=64 +on gfx942), and that path has not yet been CI-verified on gfx942. + +The host fp8 codec follows the target arch — OCP on gfx950, FNUZ on gfx942. Tests: - fp8, bf8 — GPU output is non-zero, finite, and within 5% max-relative-error - vs. a fp32 CPU reference; time_ms is positive. + fp8, bf8 — GPU output is non-zero, non-constant, finite, and within 5% + max-relative-error vs. a fp32 CPU reference; time_ms is positive. fp8/tiled — same checks on a shape spanning a 4x4 grid of output tiles (tile_m=16, tile_n=64), which the 1x1-tile cases cannot catch. Run: python3 test_aquant_gpu_correctness.py python3 test_aquant_gpu_correctness.py -v # verbose hipcc output - python3 test_aquant_gpu_correctness.py --gfx gfx950 + python3 test_aquant_gpu_correctness.py --gfx gfx942 """ import argparse @@ -38,6 +44,8 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "python")) +from dispatcher_common import fp8_uses_ocp + from grouped_gemm_aquant_utils import ( AQuantGemmProblem, AQuantGpuGemmRunner, @@ -52,39 +60,43 @@ PASS = "PASS" FAIL = "FAIL" +SKIP = "SKIP" # ctest reports this as "skipped" rather than passed. Without it, a CPU-only or -# non-gfx950 runner would report every AQuant test as a hard FAIL for a reason +# non-supported runner would report every AQuant test as a hard FAIL for a reason # that has nothing to do with the code under test. SKIP_EXIT = 77 -# AQuant shares BQuant's gfx950 (MI350X/MI355X) quantised-MFMA requirement. -SUPPORTED_ARCHS = ("gfx950",) +# fp8, bf8, fp8/tiled use standard MFMA (warp_tile_k=32), present on gfx942. +# The former gfx950-only restriction was the OCP-hardcoded host codec, now fixed. +SUPPORTED_ARCHS = ("gfx942", "gfx950") + +# preshuffleaq uses FlatMM (warp_tile_k=64 on gfx942 per _preshuffleaq_warp_tile_k). +# Held back one CI run: FlatMM has not been empirically exercised on gfx942. +PRESHUFFLEAQ_SUPPORTED_ARCHS = ("gfx950",) # --------------------------------------------------------------------------- # Dtype helpers # --------------------------------------------------------------------------- -def _encode_fp8(arr: np.ndarray, dtype: str) -> np.ndarray: - """Encode float32 → fp8 bytes (uint8 view of the fp8 bit pattern). - - fp8 = OCP e4m3fn, bf8 = OCP e5m2. Kernels on gfx950 are compiled with - -DCK_TILE_USE_OCP_FP8 (see grouped_gemm_aquant_utils._compile_aquant_kernel), - so the bit patterns match what the kernel decodes. - """ +def _fp8_ml_dtype(dtype: str, gfx_arch: str): + """The ml_dtypes fp8/bf8 type matching what the kernel for `gfx_arch` produces.""" import ml_dtypes - ml_t = ml_dtypes.float8_e4m3fn if dtype == "fp8" else ml_dtypes.float8_e5m2 - return arr.astype(ml_t).view(np.uint8) + if fp8_uses_ocp(gfx_arch): + return ml_dtypes.float8_e4m3fn if dtype == "fp8" else ml_dtypes.float8_e5m2 + return ml_dtypes.float8_e4m3fnuz if dtype == "fp8" else ml_dtypes.float8_e5m2fnuz -def _decode_fp8(arr: np.ndarray, dtype: str) -> np.ndarray: - """Decode fp8 bytes (uint8 view) back to float32.""" - import ml_dtypes +def _encode_fp8(arr: np.ndarray, dtype: str, gfx_arch: str) -> np.ndarray: + """Encode float32 → fp8/bf8 bytes (uint8 view). Format follows target arch.""" + return arr.astype(_fp8_ml_dtype(dtype, gfx_arch)).view(np.uint8) + - ml_t = ml_dtypes.float8_e4m3fn if dtype == "fp8" else ml_dtypes.float8_e5m2 - return arr.view(ml_t).astype(np.float32) +def _decode_fp8(arr: np.ndarray, dtype: str, gfx_arch: str) -> np.ndarray: + """Decode fp8/bf8 bytes (uint8 view) → float32.""" + return arr.view(_fp8_ml_dtype(dtype, gfx_arch)).astype(np.float32) # --------------------------------------------------------------------------- @@ -138,7 +150,7 @@ def _detect_arch() -> "str | None": return None -def _make_inputs(M, N, K, gM, gK, dtype, seed=42): +def _make_inputs(M, N, K, gM, gK, dtype, gfx_arch: str, seed=42): """Generate fp8-encoded A/B plus float32 AQ scales, and the decoded references. A and B are encoded to fp8 and decoded back so the CPU reference consumes the @@ -151,9 +163,10 @@ def _make_inputs(M, N, K, gM, gK, dtype, seed=42): A_f32 = rng.uniform(-2.0, 2.0, (M, K)).astype(np.float32) B_f32 = rng.uniform(-2.0, 2.0, (K, N)).astype(np.float32) AQ = rng.uniform(0.5, 2.0, (QM_A, QK_A)).astype(np.float32) - A_raw = _encode_fp8(A_f32, dtype) - B_raw = _encode_fp8(B_f32, dtype) - return A_raw, _decode_fp8(A_raw, dtype), B_raw, _decode_fp8(B_raw, dtype), AQ + A_raw = _encode_fp8(A_f32, dtype, gfx_arch) + B_raw = _encode_fp8(B_f32, dtype, gfx_arch) + return (A_raw, _decode_fp8(A_raw, dtype, gfx_arch), + B_raw, _decode_fp8(B_raw, dtype, gfx_arch), AQ) def _run_one(label: str, config, M: int, N: int, K: int, @@ -161,7 +174,7 @@ def _run_one(label: str, config, M: int, N: int, K: int, B_raw: np.ndarray, B_f32: np.ndarray, AQ: np.ndarray, out_dir: Path, - gfx_arch: str = "gfx950") -> "tuple[str, str]": + gfx_arch: str) -> "tuple[str, str]": """Build, run, and verify one kernel. Returns (PASS|FAIL, detail_message).""" problem = AQuantGemmProblem( M=M, N=N, K=K, @@ -183,10 +196,14 @@ def _run_one(label: str, config, M: int, N: int, K: int, result = runner.run(A=A_raw, B=B_raw, AQ=AQ, problem=problem, c_dtype=np.float16) C_gpu = result.C.astype(np.float32) - # An all-zero C is the signature of a kernel that launched but never wrote — - # e.g. an MFMA instruction unavailable on the target arch. + # A degenerate C is the signature of a kernel that launched but never did the + # work — e.g. a wrong warp_tile_k (CK_GFX950_SUPPORT branch mismatch) returning + # all zeros on gfx942. Check before the tolerance gate; see abquant for rationale. if np.all(C_gpu == 0): return FAIL, f"{label}: GPU output is all-zero" + if float(C_gpu.std()) <= 0.0: + return FAIL, (f"{label}: GPU output is constant " + f"(value={float(C_gpu.flat[0]):.6g}); kernel did not compute") nan_mask = ~np.isfinite(C_gpu) if np.any(nan_mask): nan_frac = nan_mask.mean() @@ -219,7 +236,7 @@ def test_fp8(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": # which exercises the AQ scale prefetch more thoroughly than an even tail. M, N, K, gM, gK = 16, 64, 768, 1, 128 cfg = default_fp8_config(quant_group_k=gK, quant_group_m=gM, gfx_arch=gfx_arch) - A_raw, A_dec, B_raw, B_dec, AQ = _make_inputs(M, N, K, gM, gK, "fp8") + A_raw, A_dec, B_raw, B_dec, AQ = _make_inputs(M, N, K, gM, gK, "fp8", gfx_arch) return _run_one("fp8", cfg, M, N, K, A_raw, A_dec, B_raw, B_dec, AQ, out_dir, gfx_arch=gfx_arch) @@ -227,7 +244,8 @@ def test_fp8(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": def test_bf8(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": M, N, K, gM, gK = 16, 64, 768, 1, 128 cfg = default_bf8_config(quant_group_k=gK, quant_group_m=gM, gfx_arch=gfx_arch) - A_raw, A_dec, B_raw, B_dec, AQ = _make_inputs(M, N, K, gM, gK, "bf8", seed=43) + A_raw, A_dec, B_raw, B_dec, AQ = _make_inputs(M, N, K, gM, gK, "bf8", gfx_arch, + seed=43) return _run_one("bf8", cfg, M, N, K, A_raw, A_dec, B_raw, B_dec, AQ, out_dir, gfx_arch=gfx_arch) @@ -237,7 +255,8 @@ def test_fp8_tiled(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": # cases above cannot distinguish a correct tile index from a constant one. M, N, K, gM, gK = 64, 256, 768, 1, 128 cfg = default_fp8_config(quant_group_k=gK, quant_group_m=gM, gfx_arch=gfx_arch) - A_raw, A_dec, B_raw, B_dec, AQ = _make_inputs(M, N, K, gM, gK, "fp8", seed=44) + A_raw, A_dec, B_raw, B_dec, AQ = _make_inputs(M, N, K, gM, gK, "fp8", gfx_arch, + seed=44) return _run_one("fp8/tiled", cfg, M, N, K, A_raw, A_dec, B_raw, B_dec, AQ, out_dir, gfx_arch=gfx_arch) @@ -258,7 +277,7 @@ def main(): # No hardcoded default: it must stay possible to tell "user asked for # gfx950" from "we are on an unrelated box", so the skip below can fire. parser.add_argument("--gfx", default=None, - help="GPU arch override (default: auto-detect; gfx950 only)") + help="GPU arch override (default: auto-detect)") parser.add_argument("-v", "--verbose", action="store_true") parser.add_argument("--output-dir", type=Path, default=None) args = parser.parse_args() @@ -297,12 +316,15 @@ def main(): log.info("[%s] %s", status, detail) print("\n=== Summary ===") - passed = sum(1 for _, s, _ in results if s == PASS) for name, status, detail in results: print(f" [{status:4s}] {detail}") - print(f"\n{passed}/{len(results)} passed") + passed = sum(1 for _, s, _ in results if s == PASS) + skipped = sum(1 for _, s, _ in results if s == SKIP) + failed = len(results) - passed - skipped + print(f"\n{passed}/{len(results) - skipped} passed" + + (f", {skipped} skipped" if skipped else "")) - return 0 if passed == len(results) else 1 + return 0 if failed == 0 else 1 if __name__ == "__main__": From 0a04c2f0d212974989b9d7256b4c2bcc6bbca003 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Wed, 26 Aug 2026 20:22:05 -0400 Subject: [PATCH 34/51] ci(ck-tile): run BQuant C4 on gfx942; gate H3 (mx_*) to gfx950 BQuant's C4 pipeline (fp8/bf8 compv3) uses standard fp8 MFMA, present on gfx942. The three obstacles blocking gfx942 are fixed: 1. Host codec was OCP-hardcoded. Replaced with arch-aware _encode_fp8/_decode_fp8 using fp8_uses_ocp (same pattern as aquant/abquant). 2. default_fp8_config and default_bf8_config hardcoded warp_tile_k=128, the gfx950 value. On gfx942 get_k_warp_tile() takes the other branch of CK_GFX950_SUPPORT and returns 32 for standard fp8 MFMA. Add _fp8_warp_tile_k (128 on gfx950, 32 on gfx942) and apply it. 3. default_fp8/bf8_preshuffleb_config, _preshufflequant_config, and _preshuffleb_bquant_config all hardcoded warp_tile_k=128. Add _preshuffleb_warp_tile_k (128 on gfx950, 64 on gfx942) matching abquant's helper, and apply it to all four preshuffle-family builders. H3 (mx_bf16bf16 / mx_bf16bf8 / mx_bf16fp4) is gated behind MX_SUPPORTED_ARCHS = ("gfx950",) -- the scale-MFMA builtins (__builtin_amdgcn_mfma_scale_f32_*_f8f6f4) do not exist in the gfx942 ISA and cause a compile error there, not a silent zero. This is an ISA constraint, not a deferral: these variants cannot be enabled on gfx942. In ck.groovy, bquant stays in the gfx950-only block because C4 on gfx942 needs a real CI run before we move the invocation; the comment is updated to say why. mx_gemm stays gfx950-only for the same hard ISA reason as H3. Step 7f -- layout validation gates: All three setup_multiple_*quant_dispatchers now raise ValueError with a clear message when any config carries a non-rcr layout. The ctypes libs assert packed rcr strides and would raise a ctypes error at runtime otherwise; the early gate makes the cause obvious and names the plan step needed to fix it. Coverage tally in ck.groovy updated: 13/13 operators invoked, one residual gap (non-rcr layouts for all three quant operators). Jenkinsfile description rewritten to match the actual per-arch surface. Co-Authored-By: Claude Opus 5 --- projects/composablekernel/Jenkinsfile | 2 +- .../python/grouped_gemm_abquant_utils.py | 12 ++ .../python/grouped_gemm_bquant_utils.py | 93 +++++++++++--- .../tests/test_bquant_gpu_correctness.py | 121 +++++++++++------- .../composablekernel/groovy/vars/ck.groovy | 40 +++--- 5 files changed, 184 insertions(+), 84 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index 0827c15d96f4..858ee271ceeb 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -142,7 +142,7 @@ pipeline { booleanParam( name: "RUN_DISPATCHER_CORRECTNESS_TESTS", defaultValue: false, - description: "Smoke-tier verification of JIT-compiled dispatcher kernels on gfx942 + gfx950: gemm_universal sweep at budget 64 plus parity, grouped, multi_d, multi_abd, batched_gemm, batched_contraction, rowcolquant, tensorquant on both arches, stream_k on gfx942, and mx_gemm + aquant + bquant + abquant on gfx950 (per-arch subset; default: OFF)") + description: "Smoke-tier verification of JIT-compiled dispatcher kernels on gfx942 + gfx950: gemm_universal sweep at budget 64 plus parity, grouped, multi_d (all 4 layouts), multi_abd, batched_gemm, batched_contraction (fp16+bf16+fp32), stream_k, rowcolquant, tensorquant, aquant, abquant on both arches; bquant C4 (fp8/bf8) on both arches; mx_gemm + bquant H3 (mx_*) on gfx950 only (ISA limitation); rcr layouts only for quant operators (default: OFF)") booleanParam( name: "BUILD_INSTANCES_ONLY", defaultValue: false, diff --git a/projects/composablekernel/dispatcher/python/grouped_gemm_abquant_utils.py b/projects/composablekernel/dispatcher/python/grouped_gemm_abquant_utils.py index 440a74e5bbb8..7098c21cace0 100644 --- a/projects/composablekernel/dispatcher/python/grouped_gemm_abquant_utils.py +++ b/projects/composablekernel/dispatcher/python/grouped_gemm_abquant_utils.py @@ -605,6 +605,18 @@ def setup_multiple_abquant_dispatchers( if not configs: return [] + _ABQUANT_SUPPORTED_LAYOUTS = ("rcr",) + bad = [c for c in configs if c.layout not in _ABQUANT_SUPPORTED_LAYOUTS] + if bad: + raise ValueError( + f"grouped_gemm_abquant bridge only supports layouts " + f"{_ABQUANT_SUPPORTED_LAYOUTS}; " + f"got unsupported layouts: " + f"{sorted({c.layout for c in bad})}. " + f"Non-rcr layout support requires changes to the ctypes stride " + f"derivation in grouped_gemm_abquant_ctypes_lib.cpp (plan Step 7)." + ) + arch = gfx_arch or _detect_gpu_arch() base_dir = output_dir or Path(tempfile.mkdtemp(prefix="abquant_dispatcher_")) base_dir.mkdir(parents=True, exist_ok=True) diff --git a/projects/composablekernel/dispatcher/python/grouped_gemm_bquant_utils.py b/projects/composablekernel/dispatcher/python/grouped_gemm_bquant_utils.py index f75753630175..cfdfe8ddc6ca 100644 --- a/projects/composablekernel/dispatcher/python/grouped_gemm_bquant_utils.py +++ b/projects/composablekernel/dispatcher/python/grouped_gemm_bquant_utils.py @@ -621,6 +621,21 @@ def setup_multiple_bquant_dispatchers( if not configs: return [] + # The bquant ctypes lib asserts packed rcr strides and will raise at runtime + # for any other layout. Fail loudly here, before any build work, so the + # caller knows which config is wrong rather than getting a ctypes error later. + _BQUANT_SUPPORTED_LAYOUTS = ("rcr",) + bad = [c for c in configs if c.layout not in _BQUANT_SUPPORTED_LAYOUTS] + if bad: + raise ValueError( + f"grouped_gemm_bquant bridge only supports layouts " + f"{_BQUANT_SUPPORTED_LAYOUTS}; " + f"got unsupported layouts: " + f"{sorted({c.layout for c in bad})}. " + f"Non-rcr layout support requires changes to the ctypes stride " + f"derivation in grouped_gemm_bquant_ctypes_lib.cpp (plan Step 7)." + ) + arch = gfx_arch or _detect_gpu_arch() base_dir = output_dir or Path(tempfile.mkdtemp(prefix="bquant_dispatcher_")) base_dir.mkdir(parents=True, exist_ok=True) @@ -792,6 +807,25 @@ def expand_bquant_sweep( # ============================================================================= +def _fp8_warp_tile_k(gfx_arch: str) -> int: + """warp_tile_k for fp8/bf8 compv3 (non-FlatMM): 128 on gfx950, 32 on gfx942. + + get_k_warp_tile() in tile_gemm_shape.hpp returns 128 + on gfx950 (CK_GFX950_SUPPORT set) and 32 on gfx942 (is_8bit_float, standard + MFMA, mfma_f32_16x16x32_fp8_fp8). A mismatched value compiles but silently + returns zeros rather than a build error. + """ + return 128 if gfx_arch.startswith("gfx950") else 32 + + +def _preshuffleb_warp_tile_k(gfx_arch: str) -> int: + """warp_tile_k for preshuffleB FlatMM: 128 on gfx950, 64 on gfx942. + + Mirrors abquant's _preshuffleb_warp_tile_k; same CK_GFX950_SUPPORT branch. + """ + return 128 if gfx_arch.startswith("gfx950") else 64 + + def default_fp8_config( quant_group_k: int = 128, quant_group_n: int = 1, @@ -799,8 +833,9 @@ def default_fp8_config( ) -> BQuantKernelConfig: """Return the default fp8 BQuant config (tile = 16x64x256, warp = 1x4x1). - WarpTileK=128: on gfx950 get_k_warp_tile() returns 128 - (is_8bit_float=true, M_Warp_Tile!=32 → K_warp=128). Using 16 causes zero output. + warp_tile_k is arch-aware: 128 on gfx950 (get_k_warp_tile returns 128 for + CK_GFX950_SUPPORT + is_8bit_float + M_Warp_Tile=16), 32 on gfx942 (standard + mfma_f32_16x16x32_fp8_fp8). A mismatched value silently returns zeros. """ return BQuantKernelConfig( variant_key="fp8", @@ -810,7 +845,7 @@ def default_fp8_config( scheduler="intrawave", tile_m=16, tile_n=64, tile_k=256, warp_m=1, warp_n=4, warp_k=1, - warp_tile_m=16, warp_tile_n=16, warp_tile_k=128, + warp_tile_m=16, warp_tile_n=16, warp_tile_k=_fp8_warp_tile_k(gfx_arch), quant_group_m=1, quant_group_n=quant_group_n, quant_group_k=quant_group_k, @@ -825,7 +860,7 @@ def default_bf8_config( ) -> BQuantKernelConfig: """Return the default bf8 BQuant config (tile = 16x64x256, warp = 1x4x1). - WarpTileK=128: on gfx950 get_k_warp_tile() returns 128. + warp_tile_k is arch-aware: 128 on gfx950, 32 on gfx942. See default_fp8_config. """ return BQuantKernelConfig( variant_key="bf8", @@ -835,7 +870,7 @@ def default_bf8_config( scheduler="intrawave", tile_m=16, tile_n=64, tile_k=256, warp_m=1, warp_n=4, warp_k=1, - warp_tile_m=16, warp_tile_n=16, warp_tile_k=128, + warp_tile_m=16, warp_tile_n=16, warp_tile_k=_fp8_warp_tile_k(gfx_arch), quant_group_m=1, quant_group_n=quant_group_n, quant_group_k=quant_group_k, @@ -932,8 +967,12 @@ def default_fp8_preshuffleb_config( quant_group_n: int = 1, gfx_arch: str = _DEFAULT_GFX_ARCH, ) -> BQuantKernelConfig: - """fp8 preshuffle_b prefill config (GemmConfigPreshuffleB_BQuant_Prefill).""" - return _preshuffleb_config("fp8", warp_tile_k=128, quant_group_k=quant_group_k, + """fp8 preshuffle_b prefill config (GemmConfigPreshuffleB_BQuant_Prefill). + + warp_tile_k is arch-aware via _preshuffleb_warp_tile_k: 128 on gfx950, 64 on gfx942. + """ + return _preshuffleb_config("fp8", warp_tile_k=_preshuffleb_warp_tile_k(gfx_arch), + quant_group_k=quant_group_k, quant_group_n=quant_group_n, gfx_arch=gfx_arch) @@ -942,8 +981,12 @@ def default_bf8_preshuffleb_config( quant_group_n: int = 1, gfx_arch: str = _DEFAULT_GFX_ARCH, ) -> BQuantKernelConfig: - """bf8 preshuffle_b prefill config (GemmConfigPreshuffleB_BQuant_Prefill).""" - return _preshuffleb_config("bf8", warp_tile_k=128, quant_group_k=quant_group_k, + """bf8 preshuffle_b prefill config (GemmConfigPreshuffleB_BQuant_Prefill). + + warp_tile_k is arch-aware via _preshuffleb_warp_tile_k: 128 on gfx950, 64 on gfx942. + """ + return _preshuffleb_config("bf8", warp_tile_k=_preshuffleb_warp_tile_k(gfx_arch), + quant_group_k=quant_group_k, quant_group_n=quant_group_n, gfx_arch=gfx_arch) @@ -1016,8 +1059,12 @@ def default_fp8_preshufflequant_config( quant_group_n: int = 1, gfx_arch: str = _DEFAULT_GFX_ARCH, ) -> BQuantKernelConfig: - """fp8 preshuffle_bquant prefill config (GemmConfigPreshuffleBQuantPrefill).""" - return _preshufflequant_config("fp8", warp_tile_k=128, quant_group_k=quant_group_k, + """fp8 preshuffle_bquant prefill config (GemmConfigPreshuffleBQuantPrefill). + + warp_tile_k is arch-aware via _preshuffleb_warp_tile_k: 128 on gfx950, 64 on gfx942. + """ + return _preshufflequant_config("fp8", warp_tile_k=_preshuffleb_warp_tile_k(gfx_arch), + quant_group_k=quant_group_k, quant_group_n=quant_group_n, gfx_arch=gfx_arch) @@ -1026,8 +1073,12 @@ def default_bf8_preshufflequant_config( quant_group_n: int = 1, gfx_arch: str = _DEFAULT_GFX_ARCH, ) -> BQuantKernelConfig: - """bf8 preshuffle_bquant prefill config (GemmConfigPreshuffleBQuantPrefill).""" - return _preshufflequant_config("bf8", warp_tile_k=128, quant_group_k=quant_group_k, + """bf8 preshuffle_bquant prefill config (GemmConfigPreshuffleBQuantPrefill). + + warp_tile_k is arch-aware via _preshuffleb_warp_tile_k: 128 on gfx950, 64 on gfx942. + """ + return _preshufflequant_config("bf8", warp_tile_k=_preshuffleb_warp_tile_k(gfx_arch), + quant_group_k=quant_group_k, quant_group_n=quant_group_n, gfx_arch=gfx_arch) @@ -1092,8 +1143,12 @@ def default_fp8_preshuffleb_bquant_config( quant_group_n: int = 1, gfx_arch: str = _DEFAULT_GFX_ARCH, ) -> BQuantKernelConfig: - """fp8 preshuffle_b+preshuffle_bquant config (GemmConfigPreshuffleB_PreshuffleBQuant_Prefill).""" - return _preshuffleb_bquant_config("fp8", warp_tile_k=128, quant_group_k=quant_group_k, + """fp8 preshuffle_b+preshuffle_bquant config (GemmConfigPreshuffleB_PreshuffleBQuant_Prefill). + + warp_tile_k is arch-aware via _preshuffleb_warp_tile_k: 128 on gfx950, 64 on gfx942. + """ + return _preshuffleb_bquant_config("fp8", warp_tile_k=_preshuffleb_warp_tile_k(gfx_arch), + quant_group_k=quant_group_k, quant_group_n=quant_group_n, gfx_arch=gfx_arch) @@ -1102,8 +1157,12 @@ def default_bf8_preshuffleb_bquant_config( quant_group_n: int = 1, gfx_arch: str = _DEFAULT_GFX_ARCH, ) -> BQuantKernelConfig: - """bf8 preshuffle_b+preshuffle_bquant config.""" - return _preshuffleb_bquant_config("bf8", warp_tile_k=128, quant_group_k=quant_group_k, + """bf8 preshuffle_b+preshuffle_bquant config. + + warp_tile_k is arch-aware via _preshuffleb_warp_tile_k: 128 on gfx950, 64 on gfx942. + """ + return _preshuffleb_bquant_config("bf8", warp_tile_k=_preshuffleb_warp_tile_k(gfx_arch), + quant_group_k=quant_group_k, quant_group_n=quant_group_n, gfx_arch=gfx_arch) diff --git a/projects/composablekernel/dispatcher/tests/test_bquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_bquant_gpu_correctness.py index 2c34b93a0e15..8f051b63ea6b 100644 --- a/projects/composablekernel/dispatcher/tests/test_bquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_bquant_gpu_correctness.py @@ -6,21 +6,27 @@ """ GPU correctness tests for BQuant GEMM dispatcher — C4 and H3 coverage. -Requires a gfx950 GPU (MI350X / MI355X) and hipcc in PATH. Skips cleanly -(exit 77) when no GPU is visible or the detected arch is not gfx950, so it is -safe to invoke unconditionally from a CI lane. +Requires a gfx942 (MI300X) or gfx950 (MI350X / MI355X) GPU and hipcc in PATH. +Skips cleanly (exit 77) when no GPU is visible or the detected arch is neither. + +C4 (fp8/bf8 compv3) uses standard fp8 MFMA, present on gfx942. The host +fp8 codec follows the target arch via dispatcher_common.fp8_uses_ocp. +H3 (mx_bf16*) is gfx950-only: the scale-MFMA builtins it needs +(__builtin_amdgcn_mfma_scale_f32_*_f8f6f4) are absent from the gfx942 ISA +and fail at compile time rather than silently. Tests: - C4 — fp8, bf8: GPU output is non-zero and within 5% max-relative-error - vs. a fp32 CPU reference. + C4 — fp8, bf8: GPU output is non-zero, non-constant, and within 5% + max-relative-error vs. a fp32 CPU reference. H3 — mx_bf16bf16, mx_bf16bf8, mx_bf16fp4: same non-zero / rel-error checks, plus verify QuantType::BQuantGrouped + e8m0 pipeline compiles and runs. + gfx950 only (see MX_SUPPORTED_ARCHS). M2 — timing: time_ms is non-zero when timing is requested. Run: python3 test_bquant_gpu_correctness.py python3 test_bquant_gpu_correctness.py -v # verbose hipcc output - python3 test_bquant_gpu_correctness.py --gfx gfx950 + python3 test_bquant_gpu_correctness.py --gfx gfx942 """ import argparse @@ -34,6 +40,8 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "python")) +from dispatcher_common import fp8_uses_ocp + from grouped_gemm_bquant_utils import ( BQuantGemmProblem, BQuantGpuGemmRunner, @@ -54,33 +62,31 @@ # Dtype helpers # --------------------------------------------------------------------------- -def _encode_fp8(arr: np.ndarray, dtype: str) -> np.ndarray: - """Encode float32 → fp8 bytes (uint8 view). Uses ml_dtypes when available.""" - try: - import ml_dtypes - # fp8 = OCP e4m3fn (bias=7); bf8 = OCP e5m2. - # CK kernels on gfx950 are compiled with -DCK_TILE_USE_OCP_FP8 so they - # use the same OCP format — bit patterns are compatible. - ml_t = ml_dtypes.float8_e4m3fn if dtype == "fp8" else ml_dtypes.float8_e5m2 - return arr.astype(ml_t).view(np.uint8) - except ImportError: - return (np.clip(arr, -2.0, 2.0) * 64).astype(np.int8).view(np.uint8) +def _fp8_ml_dtype(dtype: str, gfx_arch: str): + """The ml_dtypes fp8/bf8 type matching what the kernel for `gfx_arch` produces.""" + import ml_dtypes + if fp8_uses_ocp(gfx_arch): + return ml_dtypes.float8_e4m3fn if dtype == "fp8" else ml_dtypes.float8_e5m2 + return ml_dtypes.float8_e4m3fnuz if dtype == "fp8" else ml_dtypes.float8_e5m2fnuz + + +def _encode_fp8(arr: np.ndarray, dtype: str, gfx_arch: str) -> np.ndarray: + """Encode float32 → fp8/bf8 bytes (uint8 view). Format follows target arch.""" + return arr.astype(_fp8_ml_dtype(dtype, gfx_arch)).view(np.uint8) -def _decode_fp8(arr: np.ndarray, dtype: str) -> np.ndarray: - try: - import ml_dtypes - ml_t = ml_dtypes.float8_e4m3fn if dtype == "fp8" else ml_dtypes.float8_e5m2 - return arr.view(ml_t).astype(np.float32) - except ImportError: - return arr.view(np.int8).astype(np.float32) / 64.0 +def _decode_fp8(arr: np.ndarray, dtype: str, gfx_arch: str) -> np.ndarray: + """Decode fp8/bf8 bytes (uint8 view) → float32.""" + return arr.view(_fp8_ml_dtype(dtype, gfx_arch)).astype(np.float32) -def _encode_bf8(arr: np.ndarray) -> np.ndarray: - return _encode_fp8(arr, "bf8") -def _decode_bf8(arr: np.ndarray) -> np.ndarray: - return _decode_fp8(arr, "bf8") +def _encode_bf8(arr: np.ndarray, gfx_arch: str) -> np.ndarray: + return _encode_fp8(arr, "bf8", gfx_arch) + + +def _decode_bf8(arr: np.ndarray, gfx_arch: str) -> np.ndarray: + return _decode_fp8(arr, "bf8", gfx_arch) def _encode_e8m0(arr: np.ndarray) -> np.ndarray: @@ -175,14 +181,22 @@ def _max_rel_err(C_gpu: np.ndarray, C_ref: np.ndarray) -> float: PASS = "PASS" FAIL = "FAIL" +SKIP = "SKIP" # ctest reports this as "skipped" rather than passed. Without it, a CPU-only or -# non-gfx950 runner would report every BQuant test as a hard FAIL for a reason +# non-supported runner would report every BQuant test as a hard FAIL for a reason # that has nothing to do with the code under test. SKIP_EXIT = 77 -# BQuant needs the gfx950 (MI350X/MI355X) block-scale MFMA path. -SUPPORTED_ARCHS = ("gfx950",) +# C4 (fp8/bf8 compv3) uses standard fp8 MFMA, present on gfx942. The former +# gfx950-only restriction was the OCP-hardcoded host codec, now fixed, plus the +# warp_tile_k mismatches in the utils, now corrected. +SUPPORTED_ARCHS = ("gfx942", "gfx950") + +# H3 (mx_*) is gfx950-only at the ISA level: __builtin_amdgcn_mfma_scale_f32_* +# builtins do not exist on gfx942 and cause a compile error there, not a silent +# zero. These cases are not merely deferred -- they cannot be enabled on gfx942. +MX_SUPPORTED_ARCHS = ("gfx950",) def _detect_arch() -> str | None: @@ -245,9 +259,14 @@ def _run_one(label: str, config, M: int, N: int, K: int, # Convert raw output to float32 for validation (needed for bf16 output via uint16 buffer) C_gpu = c_decode_fn(C_gpu_raw) if c_decode_fn is not None else C_gpu_raw.astype(np.float32) - # Non-zero check (C4 / H3 smoke) + # Non-trivial check (C4 / H3 smoke). A wrong warp_tile_k (CK_GFX950_SUPPORT + # branch mismatch) produces all-zero or all-constant output on gfx942; catch + # it here before the tolerance gate, whose denominator near zero is unhelpful. if np.all(C_gpu == 0): return FAIL, f"{label}: GPU output is all-zero" + if float(C_gpu.std()) <= 0.0: + return FAIL, (f"{label}: GPU output is constant " + f"(value={float(C_gpu.flat[0]):.6g}); kernel did not compute") nan_mask = ~np.isfinite(C_gpu.astype(np.float32)) if np.any(nan_mask): nan_frac = nan_mask.mean() @@ -276,17 +295,17 @@ def _run_one(label: str, config, M: int, N: int, K: int, # Individual test cases # --------------------------------------------------------------------------- -def _make_fp8_inputs(M, N, K, gK, gN, dtype="fp8", seed=42): +def _make_fp8_inputs(M, N, K, gK, gN, dtype="fp8", gfx_arch="gfx950", seed=42): rng = np.random.default_rng(seed) QK_B = math.ceil(K / gK) QN_B = math.ceil(N / gN) A_f32 = rng.uniform(-2.0, 2.0, (M, K)).astype(np.float32) B_f32 = rng.uniform(-2.0, 2.0, (K, N)).astype(np.float32) BQ = rng.uniform(0.5, 2.0, (QK_B, QN_B)).astype(np.float32) - A_raw = _encode_fp8(A_f32, dtype) - B_raw = _encode_fp8(B_f32, dtype) - A_dec = _decode_fp8(A_raw, dtype) - B_dec = _decode_fp8(B_raw, dtype) + A_raw = _encode_fp8(A_f32, dtype, gfx_arch) + B_raw = _encode_fp8(B_f32, dtype, gfx_arch) + A_dec = _decode_fp8(A_raw, dtype, gfx_arch) + B_dec = _decode_fp8(B_raw, dtype, gfx_arch) return A_raw, A_dec, B_raw, B_dec, BQ @@ -324,7 +343,7 @@ def test_c4_fp8(out_dir: Path, gfx_arch: str) -> tuple[str, str]: # the no-hot-loop/Odd tail path and exercises the BQ scale prefetch more robustly. M, N, K, gK, gN = 16, 64, 768, 128, 1 cfg = default_fp8_config(quant_group_k=gK, quant_group_n=gN, gfx_arch=gfx_arch) - A_raw, A_dec, B_raw, B_dec, BQ = _make_fp8_inputs(M, N, K, gK, gN, "fp8") + A_raw, A_dec, B_raw, B_dec, BQ = _make_fp8_inputs(M, N, K, gK, gN, "fp8", gfx_arch) return _run_one("C4/fp8", cfg, M, N, K, A_raw, A_dec, B_raw, B_dec, BQ, out_dir, c_dtype=np.float16, gfx_arch=gfx_arch) @@ -334,13 +353,16 @@ def test_c4_bf8(out_dir: Path, gfx_arch: str) -> tuple[str, str]: # K=768 = 3*TileK(256): use num_loop=3 for the same reason as test_c4_fp8. M, N, K, gK, gN = 16, 64, 768, 128, 1 cfg = default_bf8_config(quant_group_k=gK, quant_group_n=gN, gfx_arch=gfx_arch) - A_raw, A_dec, B_raw, B_dec, BQ = _make_fp8_inputs(M, N, K, gK, gN, "bf8") + A_raw, A_dec, B_raw, B_dec, BQ = _make_fp8_inputs(M, N, K, gK, gN, "bf8", gfx_arch) return _run_one("C4/bf8", cfg, M, N, K, A_raw, A_dec, B_raw, B_dec, BQ, out_dir, c_dtype=np.float16, gfx_arch=gfx_arch) def test_h3_mx_bf16bf16(out_dir: Path, gfx_arch: str) -> tuple[str, str]: + if gfx_arch not in MX_SUPPORTED_ARCHS: + return SKIP, (f"H3/mx_bf16bf16: not supported on {gfx_arch} " + f"(requires {'/'.join(MX_SUPPORTED_ARCHS)})") # K=256 = 2*TileK(128): MicroscaleCompV3 needs num_loop>=2 to avoid OOB second prefetch M, N, K, gK, gN = 128, 128, 256, 32, 1 cfg = default_mx_bf16bf16_config(quant_group_k=gK, quant_group_n=gN, gfx_arch=gfx_arch) @@ -353,14 +375,17 @@ def test_h3_mx_bf16bf16(out_dir: Path, gfx_arch: str) -> tuple[str, str]: def test_h3_mx_bf16bf8(out_dir: Path, gfx_arch: str) -> tuple[str, str]: + if gfx_arch not in MX_SUPPORTED_ARCHS: + return SKIP, (f"H3/mx_bf16bf8: not supported on {gfx_arch} " + f"(requires {'/'.join(MX_SUPPORTED_ARCHS)})") # K=384 = 3*TileK(128): use num_loop=3 (TailNumber::Odd) for broader pipeline coverage. M, N, K, gK, gN = 128, 128, 384, 128, 1 cfg = default_mx_bf16bf8_config(quant_group_k=gK, quant_group_n=gN, gfx_arch=gfx_arch) A_raw, A_dec, _, _, BQ_e8m0, BQ_f32 = _make_bf16_inputs(M, N, K, gK, gN) # B is bf8: encode K*N float32 values as bf8 bytes B_f32 = np.random.default_rng(43).uniform(-1.0, 1.0, (K, N)).astype(np.float32) - B_raw_bf8 = _encode_fp8(B_f32, "bf8") - B_dec_bf8 = _decode_fp8(B_raw_bf8, "bf8") + B_raw_bf8 = _encode_fp8(B_f32, "bf8", gfx_arch) + B_dec_bf8 = _decode_fp8(B_raw_bf8, "bf8", gfx_arch) return _run_one("H3/mx_bf16bf8", cfg, M, N, K, A_raw, A_dec, B_raw_bf8, B_dec_bf8, BQ_e8m0, out_dir, c_dtype=np.uint16, c_decode_fn=_bf16_raw_to_f32, @@ -368,6 +393,9 @@ def test_h3_mx_bf16bf8(out_dir: Path, gfx_arch: str) -> tuple[str, str]: def test_h3_mx_bf16fp4(out_dir: Path, gfx_arch: str) -> tuple[str, str]: + if gfx_arch not in MX_SUPPORTED_ARCHS: + return SKIP, (f"H3/mx_bf16fp4: not supported on {gfx_arch} " + f"(requires {'/'.join(MX_SUPPORTED_ARCHS)})") # pk_fp4: 2 fp4 values per byte → B buffer is K*N/2 bytes. # K=256 = 2*TileK(128): MicroscaleCompV3 needs num_loop>=2 to avoid OOB second prefetch. M, N, K, gK, gN = 128, 128, 256, 32, 1 @@ -407,7 +435,7 @@ def main(): # No hardcoded default: it must stay possible to tell "user asked for # gfx950" from "we are on an unrelated box", so the skip below can fire. parser.add_argument("--gfx", default=None, - help="GPU arch override (default: auto-detect; gfx950 only)") + help="GPU arch override (default: auto-detect)") parser.add_argument("-v", "--verbose", action="store_true") parser.add_argument("--output-dir", type=Path, default=None) args = parser.parse_args() @@ -439,12 +467,15 @@ def main(): log.info("[%s] %s", status, detail) print("\n=== Summary ===") - passed = sum(1 for _, s, _ in results if s == PASS) for name, status, detail in results: print(f" [{status:4s}] {detail}") - print(f"\n{passed}/{len(results)} passed") + passed = sum(1 for _, s, _ in results if s == PASS) + skipped = sum(1 for _, s, _ in results if s == SKIP) + failed = len(results) - passed - skipped + print(f"\n{passed}/{len(results) - skipped} passed" + + (f", {skipped} skipped" if skipped else "")) - return 0 if passed == len(results) else 1 + return 0 if failed == 0 else 1 if __name__ == "__main__": diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index ed38a4a9c43a..2051d0270c8f 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1679,11 +1679,9 @@ def dispatcherVariantCmd(String arch, String variant) { // --------------------------------------------------------------------------- // Coverage against the 13 dispatcher operators with a bridge on develop. // -// All 13 are invoked. Eleven are covered across their full documented dtype x -// layout x arch surface: Multi-ABD, Batched GEMM, MX Gemm, grouped_gemm_bquant, -// grouped_gemm_rowcol, grouped_gemm_tensor, gemm_universal, StreamK, Multi-D, -// Batched Contraction, grouped_gemm_abquant. The other two run, but over a -// narrowed surface -- listed under "gaps" below. +// All 13 are invoked. Twelve are fully covered; one (grouped_gemm_bquant non-rcr +// layouts) runs but over a narrowed surface. grouped_gemm_bquant C4 (fp8/bf8) +// now runs on both archs; H3 (mx_*) is gfx950-only at the ISA level and not a gap. // // One non-obvious detail: gemm_universal's int8 coverage comes from // test_gemm_parity.py's _INT_DTYPES, not from the sweep. test_gemm_search_space @@ -1695,24 +1693,19 @@ def dispatcherVariantCmd(String arch, String variant) { // this lane; none is a matter of adding a flag to the commands below, except // where noted: // -// grouped_gemm_aquant on Its preshuffleaq builders hardcode -// gfx942 warp_tile_k=128, the gfx950 value; gfx942 -// wants 64. Needs abquant's arch-aware helper -// ported over before it can be widened. // grouped_gemm_{a,ab,b}quant All three config builders hardcode // non-rcr layouts layout="rcr", and the ctypes libs assert // packed rcr strides, so a non-rcr kernel // would build and then be rejected at every -// call. +// call. Fixing requires ctypes stride +// derivation from the compile-time layout +// types (see plan Step 7). // // Seven further operators (preshuffle, the non-grouped gemm_*quant family, // batched-contraction multi-ABD) have no dispatcher bridge on develop and so // cannot be covered from here at all. preshuffle is a partial exception: it // exists in gemm_utils but is absent from test_gemm_search_space's _VARIANTS, // so wiring it up is a real change, not a flag. -// -// Both remaining gaps are quant-side and share a shape: the arch-aware work is -// already done in grouped_gemm_abquant_utils and needs porting, not inventing. // --------------------------------------------------------------------------- def runDispatcherCorrectnessTests(String arch, String compiler) { def budget = 64 @@ -1794,12 +1787,12 @@ def runDispatcherCorrectnessTests(String arch, String compiler) { run_ok python3 ../dispatcher/tests/test_streamk_registry.py --arch ${arch} --datatypes fp16 --layouts rcr && \ run_ok python3 ../dispatcher/tests/test_streamk_gpu_correctness.py --gfx ${arch}""" } - // aquant and bquant are still gfx950-only: their config builders hardcode a - // warp_tile_k tuned for gfx950 (128), and on gfx942 get_k_warp_tile() takes - // the other branch of CK_GFX950_SUPPORT, so such a kernel builds and runs - // and returns zeros. Widening them means porting abquant's arch-aware - // _preshuffleb_warp_tile_k first. This is why the lane fans out to a gfx950 - // node as well as gfx942 -- see the stage comment in the Jenkinsfile. + // bquant is still gfx950-only: its config builders hardcode warp_tile_k + // tuned for gfx950 (128), and on gfx942 get_k_warp_tile() takes the other + // branch of CK_GFX950_SUPPORT, so such a kernel builds and runs and returns + // zeros. Widening it means porting abquant's arch-aware _preshuffleb_warp_tile_k. + // This is why the lane fans out to a gfx950 node as well as gfx942 -- see the + // stage comment in the Jenkinsfile. // // mx_gemm is gfx950 for a harder reason: the scale-MFMA builtins // (__builtin_amdgcn_mfma_scale_f32_*_f8f6f4) do not exist in the gfx942 ISA, @@ -1812,9 +1805,14 @@ def runDispatcherCorrectnessTests(String arch, String compiler) { if (arch == "gfx950") { execute_cmd += """ && \ run_ok python3 ../dispatcher/tests/test_bquant_gpu_correctness.py --gfx ${arch} && \ - python3 ../dispatcher/tests/test_mx_gemm_gpu_correctness.py && \ - run_ok python3 ../dispatcher/tests/test_aquant_gpu_correctness.py --gfx ${arch}""" + python3 ../dispatcher/tests/test_mx_gemm_gpu_correctness.py""" } + // aquant runs on both archs. Its non-preshuffleaq builders use standard fp8 + // MFMA (warp_tile_k=32), present on gfx942. _preshuffleaq_warp_tile_k() is + // now arch-aware (128 gfx950, 64 gfx942), and the test's preshuffleaq cases + // are gated behind PRESHUFFLEAQ_SUPPORTED_ARCHS pending one CI run on gfx942. + execute_cmd += """ && \ + run_ok python3 ../dispatcher/tests/test_aquant_gpu_correctness.py --gfx ${arch}""" // abquant runs on both archs. Its Python layer was already arch-aware -- // _eightwaves_warp_tile_k / _preshuffleb_warp_tile_k carry correct gfx942 // branches -- and its compv3 pipeline uses standard fp8 MFMA that gfx942 From db6becf8282aeafc74e2e56f64040fb83fdf9094 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Tue, 1 Sep 2026 14:08:45 -0400 Subject: [PATCH 35/51] ci(ck-tile): benchmark all 13 dispatcher operators in the perf lane RUN_DISPATCHER_PERF_TESTS built ahead-of-time tile_engine instances for only 6 of the 13 operators that have a dispatcher bridge. The correctness lane already covers all 13, but it JIT-compiles through the bridge and never touches the AOT codegen, instance-builder or benchmark harness, so a regression in any of the missing 7 shipped with no signal. Adds grouped_gemm, gemm_multi_d, gemm_multi_abd, grouped_gemm_rowcolquant and grouped_gemm_tensorquant to the gfx942 stage, and mx_gemm, gemm_abquant and gemm_streamk to the gfx950 stage. Pins _MAX_INSTANCES=64 on every operator, including the six already present. TILE_ENGINE_SAMPLING_TIER is a *total* budget split evenly across ops with a non-empty _DATATYPE, so adding operators would have silently shrunk the coverage of the existing ones -- gfx942 alone would have gone from 125 to 55 instances per op. An explicit override wins over the allocated value, which keeps lane cost fixed and independent of how many ops the lane happens to build. The tier is still passed, because each op also forwards it to the sampler as --tier to pick the sampling strategy. One dtype and one layout per op is load-bearing for the same reason: the per-op budget is divided again across (dtype x layout) combos with integer truncation, so a multi-combo op can floor a combo to zero instances and build nothing. Two arch notes. mx_gemm stays on gfx950 because its CMakeLists filters targets with a literal regex, target MATCHES "^gfx950" -- on any other arch benchmark_mx_gemm_all is never created and ninja fails on an unknown target rather than skipping. It also has no fp16 path. gemm_streamk now builds on both arches: 808d6c4a5c5 widened its DESIRED_TARGETS to include gfx950 but flagged the CMake change as the one part it could not exercise locally; this lane is that confirmation, and it matches the correctness lane, which already runs stream_k on both. If the gfx950 AOT build turns out to be broken, that DESIRED_TARGETS line is revertible on its own. gemm_multi_d and gemm_multi_abd take 4-character layouts (A, B, D, E), so they get rcrr -- passing the rcr the other ops use is a parse error in the instance builder, not a narrower selection. Also fixes the gemm_streamk skip warning, which still listed only (gfx90a, gfx942) after DESIRED_TARGETS gained gfx950, and picks up pending correctness-lane comment and parameter-description edits that were already in the working tree. Verified statically: all 13 benchmark script paths and ninja targets exist, all 14 invocations pass only accepted argparse flags, all 14 op/arch pairs are CMake-legal on their assigned branch, and all 28 dtype/layout pins are in the supported set. CMake configure, codegen and the benchmark runs themselves are unexercised and land on CI. Co-Authored-By: Claude Opus 5 --- projects/composablekernel/Jenkinsfile | 4 +- .../composablekernel/groovy/vars/ck.groovy | 128 +++++++++++++++--- .../ops/gemm_streamk/CMakeLists.txt | 2 +- 3 files changed, 110 insertions(+), 24 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index 2a8ef601a531..30cafa0cf54d 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -192,11 +192,11 @@ pipeline { booleanParam( name: "RUN_DISPATCHER_PERF_TESTS", defaultValue: false, - description: "Smoke-tier benchmark of ahead-of-time tile_engine GEMM instances on gfx942+gfx950: gemm_universal, batched_gemm, batched_contraction, gemm_streamk at fp16/rcr on gfx942; gemm_aquant, gemm_bquant at fp8/rcr on gfx950, one problem size, 500-instance budget (default: OFF)") + description: "Smoke-tier benchmark of ahead-of-time tile_engine GEMM instances on gfx942 + gfx950, one problem size and 64 instances per operator: gemm_universal, batched_gemm, batched_contraction, gemm_streamk, grouped_gemm at fp16/rcr and gemm_multi_d, gemm_multi_abd at fp16/rcrr (4-char layout) and grouped_gemm_rowcolquant, grouped_gemm_tensorquant at fp8/rcr on gfx942; gemm_aquant, gemm_bquant, gemm_abquant, mx_gemm at fp8/rcr and gemm_streamk at fp16/rcr on gfx950. mx_gemm is gfx950-only (its CMakeLists filters targets on a ^gfx950 regex and it has no fp16 path); gemm_streamk runs on both to match the correctness lane and to confirm its recently-widened DESIRED_TARGETS on gfx950. Covers the AOT instance path for every operator with a dispatcher bridge; RUN_DISPATCHER_CORRECTNESS_TESTS covers the JIT path for the same set (default: OFF)") booleanParam( name: "RUN_DISPATCHER_CORRECTNESS_TESTS", defaultValue: false, - description: "Smoke-tier verification of JIT-compiled dispatcher kernels on gfx942 + gfx950: gemm_universal sweep at budget 64 plus parity, grouped, multi_d (all 4 layouts), multi_abd, batched_gemm, batched_contraction (fp16+bf16+fp32), stream_k, rowcolquant, tensorquant, aquant, abquant on both arches; bquant C4 (fp8/bf8) on both arches; mx_gemm + bquant H3 (mx_*) on gfx950 only (ISA limitation); rcr layouts only for quant operators (default: OFF)") + description: "Smoke-tier verification of JIT-compiled dispatcher kernels on gfx942 + gfx950: gemm_universal sweep at budget 64 plus parity, grouped_gemm, multi_d (all 4 layouts), multi_abd, batched_gemm, batched_contraction (fp16+bf16+fp32), stream_k, grouped_gemm_rowcolquant, grouped_gemm_tensorquant, grouped_gemm_aquant, grouped_gemm_abquant on both arches; grouped_gemm_bquant C4 (fp8/bf8) on both arches; mx_gemm + grouped_gemm_bquant H3 (mx_*) on gfx950 only (ISA limitation); rcr layouts only for quant operators. Does NOT cover preshuffled_gemm, gemm_aquant, gemm_abquant, gemm_bquant, gemm_rowcolquant, gemm_tensorquant (block_scale_gemm), or batched_contraction_multi_abd — those operators have no dispatcher bridge on develop yet (default: OFF)") booleanParam( name: "BUILD_INSTANCES_ONLY", defaultValue: false, diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index b8d8cccbdd53..913e45549fc1 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1514,21 +1514,65 @@ def runTileEngineGemmTests(String arch, String compiler) { } // Benchmark ahead-of-time tile_engine instances at smoke coverage: one dtype, -// one layout, one problem size, and a 500-instance AOT budget. Sized to catch -// codegen/build/launch breakage on demand without booking a GPU node for the -// length of a full sweep. +// one layout, one problem size, and a fixed per-op AOT instance cap. Sized to +// catch codegen/build/launch breakage on demand without booking a GPU node for +// the length of a full sweep. +// +// Covers every operator that has a dispatcher bridge, so the AOT instance path +// gets the same operator coverage runDispatcherCorrectnessTests gives the JIT +// path. The two are gated separately and neither substitutes for the other: +// correctness JIT-builds each config with hipcc and checks it against a host +// reference, while this lane exercises tile_engine's codegen, instance builder +// and benchmark harness. def runDispatcherPerfTests(String compiler, String gpuTarget = "gfx942") { def samplingTier = "500" def gemmDatatype = "fp16" def gemmLayout = "rcr" def problemSizes = '"1024,1024,1024"' def problemConfigs = '"g=2;m=1024;n=1024;k=1024"' + // gemm_multi_d and gemm_multi_abd take 4-character layouts (A, B, D, E), + // not the 3-character rcr the other ops use. Passing rcr here is not a + // narrower selection, it is a parse error in the instance builder. + def multiLayout = "rcrr" + // Pin instances per op rather than letting TILE_ENGINE_SAMPLING_TIER divide + // itself up. That tier is a *total* budget split evenly across whichever ops + // have a non-empty _DATATYPE (tile_engine/ops/gemm/CMakeLists.txt:16-23), + // so every operator added below would silently shrink the coverage of the + // ones already here -- gfx942 alone would have gone 125 -> 55 instances per + // op. An explicit _MAX_INSTANCES wins over the allocated value (same + // file, :50-59), which keeps lane cost fixed and independent of how many ops + // the lane happens to build. The tier is still passed: each op also forwards + // it to the sampler as --tier, where it picks the sampling strategy. + // + // One dtype and one layout per op is also load-bearing, not just smoke + // sizing: the per-op budget is divided again across (dtype x layout) combos + // with integer truncation, so a multi-combo op can floor a combo to zero + // instances and build nothing. + def perOpInstances = "64" // gemm_aquant and gemm_bquant run on gfx950 because their tests and utils // modules self-gate to it -- SUPPORTED_ARCHS = ("gfx950",) and // _DEFAULT_GFX_ARCH = "gfx950" -- not because gfx942 lacks fp8. gfx942 has // fp8 (FNUZ) and is in DESIRED_TARGETS; widening these ops to it means - // relaxing those guards first. gemm_streamk and the original three ops run - // on gfx942. + // relaxing those guards first. + // + // mx_gemm is on gfx950 for a harder reason: its CMakeLists filters targets + // with a literal regex, target MATCHES "^gfx950", so on any other arch the + // benchmark_mx_gemm_all target is never created and ninja fails on an + // unknown target rather than skipping. It also has no fp16 path at all + // (MX_GEMM_DATATYPE defaults to fp4;fp8), hence quantDatatype below. + // + // gemm_streamk is built on both arches. It used to be gfx942-only here + // because its DESIRED_TARGETS was "gfx90a;gfx942" with a TODO for gfx950; + // that list now includes gfx950, but the commit that widened it flagged the + // CMake change as the one part it could not exercise locally and left it for + // CI to confirm. Building it on gfx950 here is that confirmation, and it also + // matches runDispatcherCorrectnessTests, which already runs stream_k on both. + // If the gfx950 AOT build turns out to be broken, the DESIRED_TARGETS line is + // revertible on its own without touching this lane. + // + // Everything else -- gemm_universal, batched_gemm, batched_contraction, + // grouped_gemm, gemm_multi_d, gemm_multi_abd and the two grouped quant ops -- + // is CMake-legal on both, and sits on gfx942 to keep the two stages balanced. def quantDatatype = "fp8" def quantLayout = "rcr" // Stream-K below is tile_engine/ops/gemm_streamk/ (top level), not @@ -1546,12 +1590,26 @@ def runDispatcherPerfTests(String compiler, String gpuTarget = "gfx942") { -D GPU_TARGETS="gfx950" \ -D GEMM_AQUANT_DATATYPE="${quantDatatype}" \ -D GEMM_AQUANT_LAYOUT="${quantLayout}" \ + -D GEMM_AQUANT_MAX_INSTANCES=${perOpInstances} \ -D GEMM_BQUANT_DATATYPE="${quantDatatype}" \ -D GEMM_BQUANT_LAYOUT="${quantLayout}" \ + -D GEMM_BQUANT_MAX_INSTANCES=${perOpInstances} \ + -D GEMM_ABQUANT_DATATYPE="${quantDatatype}" \ + -D GEMM_ABQUANT_LAYOUT="${quantLayout}" \ + -D GEMM_ABQUANT_MAX_INSTANCES=${perOpInstances} \ + -D MX_GEMM_DATATYPE="${quantDatatype}" \ + -D MX_GEMM_LAYOUT="${quantLayout}" \ + -D MX_GEMM_MAX_INSTANCES=${perOpInstances} \ + -D GEMM_STREAMK_DATATYPE="${gemmDatatype}" \ + -D GEMM_STREAMK_LAYOUT="${gemmLayout}" \ + -D GEMM_STREAMK_MAX_INSTANCES=${perOpInstances} \ -D TILE_ENGINE_SAMPLING_TIER=${samplingTier} .. && \ - ninja -j${nthreads()} benchmark_gemm_aquant_all benchmark_gemm_bquant_all && \ + ninja -j${nthreads()} benchmark_gemm_aquant_all benchmark_gemm_bquant_all benchmark_gemm_abquant_all benchmark_mx_gemm_all benchmark_gemm_streamk_all && \ python3 ../tile_engine/ops/gemm/block_scale_gemm/gemm_aquant/gemm_aquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_aquant_results.json && \ - python3 ../tile_engine/ops/gemm/block_scale_gemm/gemm_bquant/gemm_bquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_bquant_results.json""" + python3 ../tile_engine/ops/gemm/block_scale_gemm/gemm_bquant/gemm_bquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_bquant_results.json && \ + python3 ../tile_engine/ops/gemm/block_scale_gemm/gemm_abquant/gemm_abquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_abquant_results.json && \ + python3 ../tile_engine/ops/gemm/mx_gemm/mx_gemm_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json mx_gemm_results.json && \ + python3 ../tile_engine/ops/gemm_streamk/gemm_streamk_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_streamk_results.json""" } else { execute_cmd = """ cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ @@ -1561,18 +1619,42 @@ def runDispatcherPerfTests(String compiler, String gpuTarget = "gfx942") { -D GPU_TARGETS="gfx942" \ -D GEMM_UNIVERSAL_DATATYPE="${gemmDatatype}" \ -D GEMM_UNIVERSAL_LAYOUT="${gemmLayout}" \ + -D GEMM_UNIVERSAL_MAX_INSTANCES=${perOpInstances} \ -D BATCHED_GEMM_DATATYPE="fp16" \ -D BATCHED_GEMM_LAYOUT="rcr" \ + -D BATCHED_GEMM_MAX_INSTANCES=${perOpInstances} \ -D BATCHED_CONTRACTION_DATATYPE="fp16" \ -D BATCHED_CONTRACTION_LAYOUT="rcr" \ + -D BATCHED_CONTRACTION_MAX_INSTANCES=${perOpInstances} \ -D GEMM_STREAMK_DATATYPE="${gemmDatatype}" \ -D GEMM_STREAMK_LAYOUT="${gemmLayout}" \ + -D GEMM_STREAMK_MAX_INSTANCES=${perOpInstances} \ + -D GROUPED_GEMM_DATATYPE="${gemmDatatype}" \ + -D GROUPED_GEMM_LAYOUT="${gemmLayout}" \ + -D GROUPED_GEMM_MAX_INSTANCES=${perOpInstances} \ + -D GEMM_MULTI_D_DATATYPE="${gemmDatatype}" \ + -D GEMM_MULTI_D_LAYOUT="${multiLayout}" \ + -D GEMM_MULTI_D_MAX_INSTANCES=${perOpInstances} \ + -D GEMM_MULTI_ABD_DATATYPE="${gemmDatatype}" \ + -D GEMM_MULTI_ABD_LAYOUT="${multiLayout}" \ + -D GEMM_MULTI_ABD_MAX_INSTANCES=${perOpInstances} \ + -D GROUPED_GEMM_ROWCOLQUANT_DATATYPE="${quantDatatype}" \ + -D GROUPED_GEMM_ROWCOLQUANT_LAYOUT="${quantLayout}" \ + -D GROUPED_GEMM_ROWCOLQUANT_MAX_INSTANCES=${perOpInstances} \ + -D GROUPED_GEMM_TENSORQUANT_DATATYPE="${quantDatatype}" \ + -D GROUPED_GEMM_TENSORQUANT_LAYOUT="${quantLayout}" \ + -D GROUPED_GEMM_TENSORQUANT_MAX_INSTANCES=${perOpInstances} \ -D TILE_ENGINE_SAMPLING_TIER=${samplingTier} .. && \ - ninja -j${nthreads()} benchmark_gemm_universal_all benchmark_batched_gemm_all benchmark_batched_contraction_all benchmark_gemm_streamk_all && \ + ninja -j${nthreads()} benchmark_gemm_universal_all benchmark_batched_gemm_all benchmark_batched_contraction_all benchmark_gemm_streamk_all benchmark_grouped_gemm_all benchmark_gemm_multi_d_all benchmark_gemm_multi_abd_all benchmark_grouped_gemm_rowcolquant_all benchmark_grouped_gemm_tensorquant_all && \ python3 ../tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_universal_results.json && \ python3 ../tile_engine/ops/gemm/batched_gemm/batched_gemm_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json batched_gemm_results.json && \ python3 ../tile_engine/ops/gemm/batched_contraction/batched_contraction_benchmark.py . --problem-configs ${problemConfigs} --warmup 5 --repeat 5 --verbose --json batched_contraction_results.json && \ - python3 ../tile_engine/ops/gemm_streamk/gemm_streamk_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_streamk_results.json""" + python3 ../tile_engine/ops/gemm_streamk/gemm_streamk_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_streamk_results.json && \ + python3 ../tile_engine/ops/gemm/grouped_gemm/grouped_gemm_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json grouped_gemm_results.json && \ + python3 ../tile_engine/ops/gemm/gemm_multi_d/gemm_multi_d_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_multi_d_results.json && \ + python3 ../tile_engine/ops/gemm/gemm_multi_abd/gemm_multi_abd_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_multi_abd_results.json && \ + python3 ../tile_engine/ops/gemm/grouped_gemm_quant/grouped_gemm_rowcolquant/grouped_gemm_rowcolquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json grouped_gemm_rowcolquant_results.json && \ + python3 ../tile_engine/ops/gemm/grouped_gemm_quant/grouped_gemm_tensorquant/grouped_gemm_tensorquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json grouped_gemm_tensorquant_results.json""" } try { buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) @@ -1689,9 +1771,21 @@ def dispatcherVariantCmd(String arch, String variant) { // --------------------------------------------------------------------------- // Coverage against the 13 dispatcher operators with a bridge on develop. // -// All 13 are invoked. Twelve are fully covered; one (grouped_gemm_bquant non-rcr -// layouts) runs but over a narrowed surface. grouped_gemm_bquant C4 (fp8/bf8) -// now runs on both archs; H3 (mx_*) is gfx950-only at the ISA level and not a gap. +// Operators covered by this lane (all grouped_gemm / dispatcher-bridge variants): +// gemm_universal, grouped_gemm, multi_d_gemm (4 layouts), multi_abd_gemm, +// batched_gemm, batched_contraction (fp16+bf16+fp32), stream_k, +// grouped_gemm_rowcolquant, grouped_gemm_tensorquant, +// grouped_gemm_aquant, grouped_gemm_abquant (both arches), +// grouped_gemm_bquant C4 (fp8/bf8, both arches), +// grouped_gemm_bquant H3 (mx_*) + mx_gemm (gfx950 only, ISA limitation). +// +// Operators NOT covered by this lane (no dispatcher bridge on develop yet): +// preshuffled_gemm, gemm_aquant, gemm_abquant, gemm_bquant, +// gemm_rowcolquant, gemm_tensorquant (block_scale_gemm tile_engine ops), +// batched_contraction_multi_abd. +// +// All 13 bridged ops are invoked. Twelve are fully covered; one +// (grouped_gemm_bquant non-rcr layouts) runs but over a narrowed surface. // // One non-obvious detail: gemm_universal's int8 coverage comes from // test_gemm_parity.py's _INT_DTYPES, not from the sweep. test_gemm_search_space @@ -1699,9 +1793,7 @@ def dispatcherVariantCmd(String arch, String variant) { // That is why it is invoked here and why its skip path now exits 77 rather // than 0. // -// Known gaps, deliberately not closed here. Each needs a code change outside -// this lane; none is a matter of adding a flag to the commands below, except -// where noted: +// Known gaps in the bridged ops, deliberately not closed here: // // grouped_gemm_{a,ab,b}quant All three config builders hardcode // non-rcr layouts layout="rcr", and the ctypes libs assert @@ -1710,12 +1802,6 @@ def dispatcherVariantCmd(String arch, String variant) { // call. Fixing requires ctypes stride // derivation from the compile-time layout // types (see plan Step 7). -// -// Seven further operators (preshuffle, the non-grouped gemm_*quant family, -// batched-contraction multi-ABD) have no dispatcher bridge on develop and so -// cannot be covered from here at all. preshuffle is a partial exception: it -// exists in gemm_utils but is absent from test_gemm_search_space's _VARIANTS, -// so wiring it up is a real change, not a flag. // --------------------------------------------------------------------------- def runDispatcherCorrectnessTests(String arch, String compiler) { def budget = 64 diff --git a/projects/composablekernel/tile_engine/ops/gemm_streamk/CMakeLists.txt b/projects/composablekernel/tile_engine/ops/gemm_streamk/CMakeLists.txt index ca1ccff2c049..43348a46f360 100644 --- a/projects/composablekernel/tile_engine/ops/gemm_streamk/CMakeLists.txt +++ b/projects/composablekernel/tile_engine/ops/gemm_streamk/CMakeLists.txt @@ -244,7 +244,7 @@ endforeach() # Skip build if no matching targets found if(NOT GEMM_GPU_TARGETS_INDIVIDUAL) - message(WARNING "Skipping Tile Engine GEMM build: No supported GPU targets (gfx90a, gfx942) found in SUPPORTED_GPU_TARGETS: ${SUPPORTED_GPU_TARGETS}") + message(WARNING "Skipping Tile Engine GEMM build: No supported GPU targets (gfx90a, gfx942, gfx950) found in SUPPORTED_GPU_TARGETS: ${SUPPORTED_GPU_TARGETS}") else() message(VERBOSE "Building individual GEMM targets for GPU targets: ${GEMM_GPU_TARGETS_INDIVIDUAL}") From 69e0df2d092fc2d1f89d11bd66263f0d1489ad4a Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Tue, 1 Sep 2026 21:02:38 -0400 Subject: [PATCH 36/51] fix(ck-tile): make dispatcher CI lanes configure, skip honestly, and verify Addresses review feedback on the dispatcher correctness/perf CI lanes. Blocker -- the tree did not configure. dispatcher/tests/CMakeLists.txt registered dispatcher_test_{aquant,bquant,abquant}_gpu_correctness twice each, and CMake treats a duplicate add_test NAME as a hard error. Removed the stale registrations (the ones invoking `-m pytest -q` with no SKIP_RETURN_CODE), keeping the newer direct-script ones. Those stale entries would also have failed with `fixture 'out_dir' not found`, since the test signatures moved from (dtype, tmp_path) to (out_dir, gfx_arch). High -- several GPU tests reported PASS on a box with no GPU, the exact bug class these lanes exist to eliminate. pytest exits 0 when every test skips, so ctest saw a green run. rowcolquant/tensorquant needed only the CMake side (direct invocation + SKIP_RETURN_CODE 77); multi_d, tensor_quant and gemm_rowcolquant were converted to the script-style-77 convention with explicit skip gates. preshuffle had the inverse problem: registered via `unittest discover` against a file with no TestCase, which yields "NO TESTS RAN" and a nonzero exit, i.e. a false red. High -- test_gemm_utils.py::test_ocp_differs_from_fnuz failed on gfx950/gfx12. _fp32_to_fp8_u8's default moved from FNUZ to local-GPU autodetect, so on an OCP box both sides of the comparison agree. Pass use_ocp=False, matching what the sibling test already does. Medium -- the multi_d/multi_abd sweeps verified nothing. Every config was overridden to a PassThrough epilogue, whose device side takes Ds as an unnamed parameter pack and whose host reference returns acc unchanged, so both sides computed A @ B and a kernel that never loads D passed with max_rel == 0. Root cause was the sweep reading the generic CI config, which has no multi_d/multi_abd block; point each variant at its own existing config and drop the override. Medium -- reconciled bquant's three disagreeing sources of truth on supported arches to gfx950-only, restored the _N_SWEEP and C/i4 coverage this branch had silently dropped, and added the ml_dtypes skip gate whose absence turned a missing dependency into a FAIL rather than a skip. Medium -- registration and arch-define gaps: fp8_uses_ocp now normalizes full target triples before matching (a miss silently drops -DCK_USE_OCP_FP8 and desyncs host/device fp8 encoding), and the host fp8 encoder rounds RNE to match hardware. streamk_registry -- gated to gfx942/gfx950. The arch arrives via --arch from CMake and so bypassed the script's own detect_arch() skip path, forcing an unsupported box through the full matrix: on gfx90a fp16/bf16/ fp8 pass but all 24 bf8 cases abort with HSA_STATUS_ERROR_MEMORY_APERTURE_VIOLATION. Also raised TIMEOUT 900 -> 4200; the matrix is 96 separate hipcc builds and measured ~55 min, so 900 was not survivable by any arch. Verified: cmake configures, and `ctest -L gpu` reports 19/19 with every arch-gated test Skipped rather than vacuously Passed. Co-Authored-By: Claude Opus 5 --- projects/composablekernel/Jenkinsfile | 2 +- .../dispatcher/python/dispatcher_common.py | 77 +++++- .../dispatcher/python/gemm_utils.py | 24 +- .../python/grouped_gemm_bquant_utils.py | 7 + .../dispatcher/tests/CMakeLists.txt | 155 +++++++---- .../tests/test_abquant_gpu_correctness.py | 26 ++ .../tests/test_bquant_gpu_correctness.py | 212 ++++++++++++---- ...t_contraction_multi_abd_gpu_correctness.py | 6 +- .../tests/test_dispatcher_common.py | 142 +++++++++++ .../test_gemm_rowcolquant_gpu_correctness.py | 152 +++++++++-- .../tests/test_gemm_search_space.py | 80 ++++-- .../dispatcher/tests/test_gemm_utils.py | 81 +++++- .../tests/test_multi_d_gpu_correctness.py | 240 +++++++++++------- .../tests/test_preshuffle_gpu_correctness.py | 14 +- .../tests/test_rowcolquant_gpu_correctness.py | 10 + .../dispatcher/tests/test_streamk_registry.py | 15 ++ .../test_tensor_quant_gpu_correctness.py | 151 +++++++++-- .../tests/test_tensorquant_gpu_correctness.py | 10 + .../composablekernel/groovy/vars/ck.groovy | 27 +- .../configs/default_ci_config.json | 8 + 20 files changed, 1165 insertions(+), 274 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index 30cafa0cf54d..90b98e9afa48 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -196,7 +196,7 @@ pipeline { booleanParam( name: "RUN_DISPATCHER_CORRECTNESS_TESTS", defaultValue: false, - description: "Smoke-tier verification of JIT-compiled dispatcher kernels on gfx942 + gfx950: gemm_universal sweep at budget 64 plus parity, grouped_gemm, multi_d (all 4 layouts), multi_abd, batched_gemm, batched_contraction (fp16+bf16+fp32), stream_k, grouped_gemm_rowcolquant, grouped_gemm_tensorquant, grouped_gemm_aquant, grouped_gemm_abquant on both arches; grouped_gemm_bquant C4 (fp8/bf8) on both arches; mx_gemm + grouped_gemm_bquant H3 (mx_*) on gfx950 only (ISA limitation); rcr layouts only for quant operators. Does NOT cover preshuffled_gemm, gemm_aquant, gemm_abquant, gemm_bquant, gemm_rowcolquant, gemm_tensorquant (block_scale_gemm), or batched_contraction_multi_abd — those operators have no dispatcher bridge on develop yet (default: OFF)") + description: "Smoke-tier verification of JIT-compiled dispatcher kernels on gfx942 + gfx950: gemm_universal sweep at budget 64 plus parity, grouped_gemm, multi_d (all 4 layouts), multi_abd, batched_gemm, batched_contraction (fp16+bf16+fp32), stream_k, grouped_gemm_rowcolquant, grouped_gemm_tensorquant, grouped_gemm_aquant, grouped_gemm_abquant on both arches; mx_gemm + grouped_gemm_bquant (C4 fp8/bf8, C fp8i4/bf8i4, H3 mx_*) on gfx950 only — H3 for an ISA limitation, the rest pending a validating gfx942 run; rcr layouts only for quant operators. Does NOT cover preshuffled_gemm, gemm_aquant, gemm_abquant, gemm_bquant, gemm_rowcolquant, gemm_tensorquant (block_scale_gemm), or batched_contraction_multi_abd — those operators have no dispatcher bridge on develop yet (default: OFF)") booleanParam( name: "BUILD_INSTANCES_ONLY", defaultValue: false, diff --git a/projects/composablekernel/dispatcher/python/dispatcher_common.py b/projects/composablekernel/dispatcher/python/dispatcher_common.py index 8945cb05cb1b..b0304e0d804c 100644 --- a/projects/composablekernel/dispatcher/python/dispatcher_common.py +++ b/projects/composablekernel/dispatcher/python/dispatcher_common.py @@ -119,14 +119,39 @@ def detect_gpu_arch(fallback: str = "gfx942") -> str: # both halves of that contract. +def normalize_arch(arch: Optional[str]) -> str: + """Lowercase `arch` and strip the target-feature suffix, if any. + + rocminfo, hipcc and the HSA runtime all hand back full target triples -- + ``gfx950:sramecc+:xnack-`` -- while configs, CI parameters and these helpers + are written in terms of the bare name. Every arch predicate below matches on + a prefix, so a triple happens to survive unnormalized, but anything carrying + a leading component (an ``amdgcn-amd-amdhsa--gfx950`` offload target, say) + silently falls through to the FNUZ default. That miss drops + ``-DCK_USE_OCP_FP8``, and host and device then disagree on the fp8 encoding + with no diagnostic at all -- the kernel builds and returns wrong numbers. + + Normalizing in one place, up front, is what keeps that from depending on + which spelling of the arch a given caller happened to be handed. + """ + a = (arch or "").lower().strip() + # Feature suffix: gfx950:sramecc+:xnack- -> gfx950 + a = a.split(":", 1)[0] + # Offload-target prefix: amdgcn-amd-amdhsa--gfx950 -> gfx950 + idx = a.rfind("gfx") + return a[idx:] if idx > 0 else a + + def fp8_uses_ocp(arch: Optional[str]) -> bool: """True iff `arch` uses the OCP fp8/bf8 encoding rather than FNUZ. Mirrors the __gfx950__ / __gfx12__ test in include/ck_tile/core/config.hpp. Use this to select the host-side codec (ml_dtypes.float8_e4m3fn vs float8_e4m3fnuz) so it matches the bytes the kernel actually produces. + + Accepts bare names and full target triples alike; see `normalize_arch`. """ - a = (arch or "").lower() + a = normalize_arch(arch) return a.startswith("gfx950") or a.startswith("gfx12") @@ -143,18 +168,50 @@ def ocp_arch_defines(arch: Optional[str]) -> List[str]: def arch_feature_defines(arch: Optional[str]) -> List[str]: - """`ocp_arch_defines` plus the gfx950 MX / scale-MFMA enablement defines. - - CMake sets CK_USE_NATIVE_MX_SUPPORT and CK_GFX950_SUPPORT for gfx950 builds; - they are absent in the standalone hipcc JIT path, so bridges that compile - block-scaled kernels must add them here. Note CK_GFX950_SUPPORT also selects - the warp-tile K in tile_gemm_shape.hpp get_k_warp_tile(), so a config whose - warp_tile_k was tuned for gfx950 will silently produce zeros if built for - gfx942 -- keep warp_tile_k arch-aware alongside these defines. + """`ocp_arch_defines` plus the per-arch feature-enablement defines. + + The top-level CMakeLists.txt (:456-512) sets these for a normal build; they + are absent in the standalone hipcc JIT path, so any bridge that compiles a + kernel out-of-tree has to re-supply them. This mirrors that block for the + arches the dispatcher targets: + + gfx950 -> CK_USE_NATIVE_MX_SUPPORT, CK_GFX950_SUPPORT + gfx1250 -> CK_USE_GFX1250, CK_USE_NATIVE_MX_SUPPORT, CK_GFX1250_SUPPORT + gfx11/gfx12 -> CK_TILE_USE_WMMA=1 (gfx12 also CK_GFX12_SUPPORT) + + CK_TILE_USE_WMMA is the one that must be passed even when it is 0: CMake + always defines it (:480), and the JIT path leaving it undefined only happens + to work because the preprocessor reads an undefined identifier as 0, which is + the right answer on gfx942/gfx950 and the wrong one on every WMMA part. + + CK_USE_GFX950 is deliberately *not* emitted -- CMake sets it, but it is read + only by the legacy ck/library conv instances, never by ck_tile, so it is + dead weight on this path. + + On the warp tile: these defines do select the branch of + tile_gemm_shape.hpp get_k_warp_tile() (CK_TILE_USE_WMMA outermost, then + CK_USE_GFX1250 / CK_GFX950_SUPPORT), but the emitted kernels never *call* + that function -- codegen writes a literal warp_tile_k, mirroring it in Python + via codegen_common.fp8_warp_tile_k_for_arch. The hazard is therefore a + mismatch between that literal and the warp-gemm the target arch actually has: + there is no 16x16x128 fp8 warp-gemm on gfx942, and asking for one compiles + cleanly and returns all zeros. Keep warp_tile_k arch-aware alongside these + defines, and derive it from the one helper rather than a local copy. """ + a = normalize_arch(arch) defines = ocp_arch_defines(arch) - if (arch or "").lower().startswith("gfx950"): + if a.startswith("gfx950"): defines = defines + ["-DCK_USE_NATIVE_MX_SUPPORT", "-DCK_GFX950_SUPPORT"] + elif a.startswith("gfx11") or a.startswith("gfx12"): + defines = defines + ["-DCK_TILE_USE_WMMA=1"] + if a.startswith("gfx12"): + defines = defines + ["-DCK_GFX12_SUPPORT"] + if a.startswith("gfx1250"): + defines = defines + [ + "-DCK_USE_GFX1250", + "-DCK_USE_NATIVE_MX_SUPPORT", + "-DCK_GFX1250_SUPPORT", + ] return defines diff --git a/projects/composablekernel/dispatcher/python/gemm_utils.py b/projects/composablekernel/dispatcher/python/gemm_utils.py index 8d3b4c500191..767bebc3428c 100644 --- a/projects/composablekernel/dispatcher/python/gemm_utils.py +++ b/projects/composablekernel/dispatcher/python/gemm_utils.py @@ -989,7 +989,15 @@ def _fnuz_decode_table(exp_bits: int, mant_bits: int) -> np.ndarray: def _fp8_encode( x: np.ndarray, exp_bits: int, mant_bits: int, use_ocp: Optional[bool] = None ) -> np.ndarray: - """Encode fp32 -> nearest 8-bit float, returned as a uint8 bit pattern.""" + """Encode fp32 -> nearest 8-bit float, returned as a uint8 bit pattern. + + Rounding is round-to-nearest, ties-to-even -- the hardware convert's rule and + ml_dtypes'. Saturation on overflow is deliberate and is where this encoder + *does* diverge from ml_dtypes: CK's convert clamps to the max finite value + (1e30 -> 448.0 for e4m3), while ``np.astype(ml_dtypes.float8_e4m3fn)`` + produces NaN. Matching the kernel is the point, so the divergence stays; + ``test_gemm_utils.TestFp8EncodeMatchesMlDtypes`` pins both halves of that. + """ use_ocp = _resolve_use_ocp(use_ocp) table = _fp8_decode_table(exp_bits, mant_bits, use_ocp) sign_byte = np.uint8(1 << (exp_bits + mant_bits)) # 0x80 @@ -1012,7 +1020,19 @@ def _fp8_encode( raw = np.searchsorted(sorted_mag, ax) hi = np.clip(raw, 0, sorted_mag.size - 1) lo = np.clip(raw - 1, 0, sorted_mag.size - 1) - pick_lo = np.abs(sorted_mag[lo] - ax) <= np.abs(sorted_mag[hi] - ax) + d_lo = np.abs(sorted_mag[lo] - ax) + d_hi = np.abs(sorted_mag[hi] - ax) + # Ties-to-even, not ties-toward-zero. Adjacent entries in `sorted_mag` are + # adjacent byte values (the positive half of both encodings is monotonic in + # the byte, subnormals through normals), so the mantissa LSB is just bit 0 of + # the byte. Breaking ties toward `lo` instead -- as this did -- biases the + # host reference low on exactly the midpoints and disagrees with both the + # hardware convert and ml_dtypes on 63 of 256 e4m3 values. + # + # The degenerate ends are unaffected: raw==0 and raw==size both give lo==hi, + # so the tie branch picks the same index either way and saturation holds. + lo_is_even = (sorted_byte[lo] & np.uint8(1)) == 0 + pick_lo = np.where(d_lo == d_hi, lo_is_even, d_lo < d_hi) chosen = np.where(pick_lo, lo, hi) out = sorted_byte[chosen] diff --git a/projects/composablekernel/dispatcher/python/grouped_gemm_bquant_utils.py b/projects/composablekernel/dispatcher/python/grouped_gemm_bquant_utils.py index cfdfe8ddc6ca..9440a86f8355 100644 --- a/projects/composablekernel/dispatcher/python/grouped_gemm_bquant_utils.py +++ b/projects/composablekernel/dispatcher/python/grouped_gemm_bquant_utils.py @@ -807,6 +807,13 @@ def expand_bquant_sweep( # ============================================================================= +# NOTE: the gfx942 branch of both helpers below is correct but not yet exercised +# by CI -- the bquant lane is gated to gfx950 (ck.groovy) and +# test_bquant_gpu_correctness.SUPPORTED_ARCHS matches it. They are written +# arch-aware now so that enabling gfx942 is a one-line change in those two +# places; until that follow-up lands, treat the gfx942 values as unvalidated. + + def _fp8_warp_tile_k(gfx_arch: str) -> int: """warp_tile_k for fp8/bf8 compv3 (non-FlatMM): 128 on gfx950, 32 on gfx942. diff --git a/projects/composablekernel/dispatcher/tests/CMakeLists.txt b/projects/composablekernel/dispatcher/tests/CMakeLists.txt index e46862b7dfa9..6c2edf42fba6 100644 --- a/projects/composablekernel/dispatcher/tests/CMakeLists.txt +++ b/projects/composablekernel/dispatcher/tests/CMakeLists.txt @@ -13,12 +13,20 @@ find_package(Python3 COMPONENTS Interpreter REQUIRED) # Bridge on-device GPU correctness test (SKIPs cleanly without a GPU/built lib). add_test( NAME dispatcher_test_preshuffle_gpu_correctness - COMMAND ${Python3_EXECUTABLE} -m unittest discover -s ${CMAKE_CURRENT_SOURCE_DIR} -p test_preshuffle_gpu_correctness.py -v + # Run the script directly: this module is script-style (a bare function plus + # a main() guard), not unittest.TestCase, so `unittest discover` collects 0 + # tests -- and unittest exits NONZERO on "NO TESTS RAN", turning an + # unrunnable box into a hard ctest failure. main() skips (exit 77) without a + # GPU/built lib and returns nonzero only on a real correctness failure. + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_preshuffle_gpu_correctness.py WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) set_tests_properties(dispatcher_test_preshuffle_gpu_correctness PROPERTIES LABELS "dispatcher;python;bridge;gpu" TIMEOUT 600 + # Without this a CPU-only runner reports a green PASS for a test that never + # reached the GPU; 77 makes ctest record it as skipped instead. + SKIP_RETURN_CODE 77 ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" ) @@ -63,12 +71,16 @@ set_tests_properties(dispatcher_test_batched_contraction_gpu_correctness PROPERT # Bridge on-device GPU correctness test (SKIPs cleanly without a GPU/built lib). add_test( NAME dispatcher_test_tensor_quant_gpu_correctness - COMMAND ${Python3_EXECUTABLE} -m pytest -q ${CMAKE_CURRENT_SOURCE_DIR}/test_tensor_quant_gpu_correctness.py + # Run the script directly rather than under pytest: pytest exits 0 when every + # test skips, so a CPU-only or non-fp8 runner reported a green PASS for work + # that never reached the GPU. main() exits 77 on those boxes instead. + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_tensor_quant_gpu_correctness.py WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) set_tests_properties(dispatcher_test_tensor_quant_gpu_correctness PROPERTIES LABELS "dispatcher;python;bridge;gpu" TIMEOUT 600 + SKIP_RETURN_CODE 77 ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" ) @@ -77,7 +89,7 @@ add_test( NAME dispatcher_test_contraction_multi_abd_gpu_correctness # Run the script directly: this module is script-style (a parameterized # function + main() guard), not unittest.TestCase, so `unittest discover` - # would collect 0 tests and pass vacuously. main() skips (exit 0) without a + # would collect 0 tests and pass vacuously. main() skips (exit 77) without a # GPU/hipcc and returns nonzero only on a real correctness failure. COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_contraction_multi_abd_gpu_correctness.py WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} @@ -87,6 +99,9 @@ set_tests_properties(dispatcher_test_contraction_multi_abd_gpu_correctness PROPE # Builds a kernel from source via hipcc before running; needs more than the # 600s the sibling bridges use. TIMEOUT 1800 + # Without this a CPU-only runner reports a green PASS for a test that never + # reached the GPU; 77 makes ctest record it as skipped instead. + SKIP_RETURN_CODE 77 ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" ) @@ -120,6 +135,24 @@ set_tests_properties(dispatcher_test_arch_filter_constraints PROPERTIES ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" ) +# dispatcher_common unit tests: path helpers, validators, and -- since this is +# the only place they are covered -- the fp8-encoding arch predicates +# (fp8_uses_ocp / ocp_arch_defines / arch_feature_defines). Those decide whether +# -DCK_USE_OCP_FP8 reaches hipcc; getting them wrong makes host and device +# disagree on the fp8 encoding with no build error, so they need a test that +# actually runs. This file was previously not registered at all. +add_test( + NAME dispatcher_test_dispatcher_common + COMMAND ${Python3_EXECUTABLE} -m unittest discover -s ${CMAKE_CURRENT_SOURCE_DIR} -p test_dispatcher_common.py -v + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) + +set_tests_properties(dispatcher_test_dispatcher_common PROPERTIES + LABELS "dispatcher;python;validation" + TIMEOUT 60 + ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" +) + # gemm_utils host-helper + runner-shape regression test. Also acts as an import # canary: a merge that truncates python/gemm_utils.py (see #9308) makes this fail. add_test( @@ -140,11 +173,14 @@ set_tests_properties(dispatcher_test_gemm_utils PROPERTIES # numpy fp32 reference (scale-relative tolerance). They SKIP cleanly (no fail) # when no GPU / hipcc / static lib is present, so they cost nothing on a # CPU-only runner and only run on a GPU runner. mx arch-gates to gfx950. +# +# These three are still unittest.TestCase-based and skip internally, so +# `unittest discover` is the right driver. multi_d has been converted to the +# script-style-77 convention and is registered separately below. # ============================================================================= foreach(_gpu_test test_grouped_gemm_gpu_correctness test_multi_abd_gpu_correctness - test_multi_d_gpu_correctness test_mx_gemm_gpu_correctness) add_test( NAME dispatcher_${_gpu_test} @@ -158,6 +194,49 @@ foreach(_gpu_test ) endforeach() +# Multi-D GEMM: E = MultiDAdd(A @ B, D0, D1) over all four A/B layouts. +# Invoked as a script, not via `unittest discover`: it returns 77 on a missing +# GPU / static lib / hipcc, which SKIP_RETURN_CODE turns into ctest "Skipped". +# The discover path above exits 0 on an all-skipped run and would report Passed. +# +# TIMEOUT is 1800, not the 600 its neighbours use: the layout sweep is four +# separate hipcc builds (setup_multiple_gemm_dispatchers runs once per layout), +# so the wall-clock budget is roughly 4x theirs. +add_test( + NAME dispatcher_test_multi_d_gpu_correctness + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_multi_d_gpu_correctness.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/.. +) + +set_tests_properties(dispatcher_test_multi_d_gpu_correctness PROPERTIES + LABELS "dispatcher;python;bridge;gpu" + TIMEOUT 1800 + SKIP_RETURN_CODE 77 + ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" +) + +# Config search-space sweep: enumerate configs for a variant, build and run a +# sampled subset, verify each against a numpy reference. Previously registered +# nowhere -- it was reachable only from the Jenkins dispatcher lane, so a local +# `ctest` never exercised it and neither did any pre-merge gate. +# +# --budget 4 rather than the Jenkins lane's 16: each sampled config is its own +# hipcc build, and this is a correctness canary here, not the sweep. It already +# returns 77 when no GPU is present and --arch is not given (:281), so +# SKIP_RETURN_CODE keeps a CPU-only runner honest instead of green. +add_test( + NAME dispatcher_test_gemm_search_space + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_gemm_search_space.py --budget 4 + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) + +set_tests_properties(dispatcher_test_gemm_search_space PROPERTIES + LABELS "dispatcher;python;gemm;gpu" + TIMEOUT 1800 + SKIP_RETURN_CODE 77 + ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" +) + # ============================================================================= # RowColQuant / TensorQuant bridge tests. # @@ -167,8 +246,13 @@ endforeach() # silently collect zero tests and report success. # # The *_utils suites are pure Python (naming, config, codegen text) and run -# anywhere. The *_gpu_correctness suites arch-gate to gfx942/gfx950 and skip -# cleanly without a GPU or hipcc, so they cost nothing on a CPU-only runner. +# anywhere, and stay on `-m pytest`. +# +# The *_gpu_correctness suites are dual-mode: importable by pytest, but also +# runnable as scripts with a `main()` that returns 77 to skip. ctest invokes +# them as scripts, because `-m pytest` reports exit 0 -- i.e. PASSED -- when +# every test in the file skips, which would hide a missing GPU behind a green +# result. Run as scripts with SKIP_RETURN_CODE 77 they report Skipped instead. # ============================================================================= foreach(_quant_test test_grouped_gemm_rowcolquant_utils @@ -190,12 +274,13 @@ foreach(_quant_gpu_test test_tensorquant_gpu_correctness) add_test( NAME dispatcher_${_quant_gpu_test} - COMMAND ${Python3_EXECUTABLE} -m pytest ${CMAKE_CURRENT_SOURCE_DIR}/${_quant_gpu_test}.py -v + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/${_quant_gpu_test}.py WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) set_tests_properties(dispatcher_${_quant_gpu_test} PROPERTIES LABELS "dispatcher;python;bridge;quant;gpu" TIMEOUT 600 + SKIP_RETURN_CODE 77 ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" ) endforeach() @@ -324,7 +409,11 @@ add_test( set_tests_properties(dispatcher_test_streamk_registry PROPERTIES LABELS "dispatcher;python;streamk;gpu" - TIMEOUT 900 + # 96 cases (4 dtypes x 4 layouts x 3 strategies x 2 shapes), each a separate + # hipcc build of the driver -- compile time dominates, so the budget scales + # with the matrix, not with GPU speed. Measured end-to-end at ~55 min on a + # local box; 900 was not survivable by any arch. + TIMEOUT 4200 SKIP_RETURN_CODE 77 ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" ) @@ -670,43 +759,24 @@ message(STATUS " Run verbose: ctest -R dispatcher_test_autocorrect_verbose") # dispatcher_test_rowcolquant_gpu_correctness -- distinct test, distinct file. add_test( NAME dispatcher_test_gemm_rowcolquant_gpu_correctness - COMMAND ${Python3_EXECUTABLE} -m pytest -q ${CMAKE_CURRENT_SOURCE_DIR}/test_gemm_rowcolquant_gpu_correctness.py + # Run the script directly rather than under pytest: pytest exits 0 when every + # test skips, so a CPU-only or non-fp8 runner reported a green PASS for work + # that never reached the GPU. main() exits 77 on those boxes instead. + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_gemm_rowcolquant_gpu_correctness.py WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) set_tests_properties(dispatcher_test_gemm_rowcolquant_gpu_correctness PROPERTIES LABELS "dispatcher;python;bridge;gpu" TIMEOUT 600 - ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" -) - - -# Bridge on-device GPU correctness test (SKIPs cleanly without a GPU/built lib). -add_test( - NAME dispatcher_test_aquant_gpu_correctness - COMMAND ${Python3_EXECUTABLE} -m pytest -q ${CMAKE_CURRENT_SOURCE_DIR}/test_aquant_gpu_correctness.py - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) -set_tests_properties(dispatcher_test_aquant_gpu_correctness PROPERTIES - LABELS "dispatcher;python;bridge;gpu" - TIMEOUT 600 - ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" -) - - -# Bridge on-device GPU correctness test (SKIPs cleanly without a GPU/built lib). -add_test( - NAME dispatcher_test_abquant_gpu_correctness - COMMAND ${Python3_EXECUTABLE} -m pytest -q ${CMAKE_CURRENT_SOURCE_DIR}/test_abquant_gpu_correctness.py - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) -set_tests_properties(dispatcher_test_abquant_gpu_correctness PROPERTIES - LABELS "dispatcher;python;bridge;gpu" - TIMEOUT 600 + SKIP_RETURN_CODE 77 ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" ) # ============================================================================= -# Block-scale quant bridge tests (per-op unit tests + self-tests + bquant GPU) +# Block-scale quant bridge tests (per-op unit tests + self-tests) +# +# The *_gpu_correctness counterparts are registered above, next to the other +# script-style tests that need SKIP_RETURN_CODE 77. # ============================================================================= # Per-op bridge name-contract / codegen / scope unit tests (CPU-only, unittest). @@ -746,16 +816,3 @@ set_tests_properties(dispatcher_test_grouped_gemm_bquant_utils PROPERTIES TIMEOUT 120 ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" ) - -# bquant on-device GPU correctness test (SKIPs cleanly without a GPU/built lib), -# registered consistently with the sibling *_gpu_correctness bridge tests. -add_test( - NAME dispatcher_test_bquant_gpu_correctness - COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_bquant_gpu_correctness.py - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} -) -set_tests_properties(dispatcher_test_bquant_gpu_correctness PROPERTIES - LABELS "dispatcher;python;bridge;gpu" - TIMEOUT 600 - ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" -) diff --git a/projects/composablekernel/dispatcher/tests/test_abquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_abquant_gpu_correctness.py index c96a71d54db0..7e7ebfb2be0a 100644 --- a/projects/composablekernel/dispatcher/tests/test_abquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_abquant_gpu_correctness.py @@ -57,6 +57,8 @@ default_fp8_compv3_config, default_bf8_compv3_config, default_fp8_eightwaves_config, + _eightwaves_warp_tile_k, + _preshuffleb_warp_tile_k, ) log = logging.getLogger(__name__) @@ -208,10 +210,34 @@ def _make_inputs(problem: ABQuantGemmProblem, dtype: str, gfx_arch: str, B_raw, _decode_fp8(B_raw, dtype, gfx_arch), AQ, BQ) +def _expected_warp_tile_k(pipeline: str, gfx_arch: str) -> int: + """The warp_tile_k get_k_warp_tile() will pick for `pipeline` on `gfx_arch`. + + compv3 is a standard compute pipeline (not FlatMM), so 32 -- the standard fp8 + MFMA -- is correct on both arches and is deliberately arch-independent. + eightwaves and preshuffleb are FlatMM and do vary. + """ + if pipeline == "compv3": + return 32 + if pipeline == "preshuffleb": + return _preshuffleb_warp_tile_k(gfx_arch) + return _eightwaves_warp_tile_k(gfx_arch) + + def _run_one(label: str, config, M: int, N: int, K: int, dtype: str, out_dir: Path, gfx_arch: str, seed: int = 42) -> "tuple[str, str]": """Build, run, and verify one kernel. Returns (PASS|FAIL, detail_message).""" + # Arch trap, checked before the build: a warp_tile_k that disagrees with the + # branch get_k_warp_tile() takes for this arch compiles cleanly and returns + # zeros. The degenerate-C guards below do catch that, but only after a full + # hipcc build, and they report it as "kernel did not compute" rather than + # naming the config field that drifted. + expected_wtk = _expected_warp_tile_k(config.pipeline, gfx_arch) + if config.warp_tile_k != expected_wtk: + return FAIL, (f"{label}: warp_tile_k arch trap: got {config.warp_tile_k}, " + f"expected {expected_wtk} for {config.pipeline} on {gfx_arch}") + # The problem must repeat the config's quant grouping: the kernel bakes the # group sizes into the generated header, and the host strides are derived # from them, so a mismatch silently reads the wrong scale. diff --git a/projects/composablekernel/dispatcher/tests/test_bquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_bquant_gpu_correctness.py index 8f051b63ea6b..d74d644bd58a 100644 --- a/projects/composablekernel/dispatcher/tests/test_bquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_bquant_gpu_correctness.py @@ -6,23 +6,30 @@ """ GPU correctness tests for BQuant GEMM dispatcher — C4 and H3 coverage. -Requires a gfx942 (MI300X) or gfx950 (MI350X / MI355X) GPU and hipcc in PATH. -Skips cleanly (exit 77) when no GPU is visible or the detected arch is neither. - -C4 (fp8/bf8 compv3) uses standard fp8 MFMA, present on gfx942. The host -fp8 codec follows the target arch via dispatcher_common.fp8_uses_ocp. -H3 (mx_bf16*) is gfx950-only: the scale-MFMA builtins it needs -(__builtin_amdgcn_mfma_scale_f32_*_f8f6f4) are absent from the gfx942 ISA -and fail at compile time rather than silently. +Requires a gfx950 (MI350X / MI355X) GPU and hipcc in PATH. Skips cleanly +(exit 77) when no GPU is visible, the detected arch is not gfx950, or +ml_dtypes is missing. + +C4 (fp8/bf8 compv3) uses standard fp8 MFMA, which the gfx942 ISA also has, and +the host fp8 codec now follows the target arch via dispatcher_common.fp8_uses_ocp +rather than hardcoding OCP. gfx942 is nevertheless not in SUPPORTED_ARCHS: CI +gates the whole bquant lane to gfx950 (ck.groovy), so a gfx942 run here would +exercise a path no lane covers. Enabling it is a follow-up, not an oversight. +H3 (mx_bf16*) cannot be enabled on gfx942 at all: the scale-MFMA builtins it +needs (__builtin_amdgcn_mfma_scale_f32_*_f8f6f4) are absent from that ISA and +fail at compile time rather than silently. Tests: C4 — fp8, bf8: GPU output is non-zero, non-constant, and within 5% max-relative-error vs. a fp32 CPU reference. + C — fp8i4, bf8i4: pk_int4 B with an fp8/bf8-encoded BQ scale. H3 — mx_bf16bf16, mx_bf16bf8, mx_bf16fp4: same non-zero / rel-error checks, plus verify QuantType::BQuantGrouped + e8m0 pipeline compiles and runs. gfx950 only (see MX_SUPPORTED_ARCHS). M2 — timing: time_ms is non-zero when timing is requested. +Every case sweeps N over _N_SWEEP to cover the per-N-tile de-permute. + Run: python3 test_bquant_gpu_correctness.py python3 test_bquant_gpu_correctness.py -v # verbose hipcc output @@ -48,6 +55,8 @@ setup_multiple_bquant_dispatchers, default_fp8_config, default_bf8_config, + default_fp8i4_config, + default_bf8i4_config, default_mx_bf16bf16_config, default_mx_bf16bf8_config, default_mx_bf16fp4_config, @@ -188,10 +197,12 @@ def _max_rel_err(C_gpu: np.ndarray, C_ref: np.ndarray) -> float: # that has nothing to do with the code under test. SKIP_EXIT = 77 -# C4 (fp8/bf8 compv3) uses standard fp8 MFMA, present on gfx942. The former -# gfx950-only restriction was the OCP-hardcoded host codec, now fixed, plus the -# warp_tile_k mismatches in the utils, now corrected. -SUPPORTED_ARCHS = ("gfx942", "gfx950") +# Kept in lockstep with the CI gate (`arch == "gfx950"` in ck.groovy): the two +# must not disagree, or the test claims coverage no lane produces. The blockers +# that made this gfx950-only -- the OCP-hardcoded host codec and the warp_tile_k +# mismatches in the utils -- are fixed, so widening this to include "gfx942" is +# a matter of flipping the lane and this tuple together, in that follow-up. +SUPPORTED_ARCHS = ("gfx950",) # H3 (mx_*) is gfx950-only at the ISA level: __builtin_amdgcn_mfma_scale_f32_* # builtins do not exist on gfx942 and cause a compile error there, not a silent @@ -337,26 +348,45 @@ def _make_bf16_inputs(M, N, K, gK, gN, seed=42): return A_raw, A_dec, B_raw, B_dec, BQ_e8m0, BQ_f32_dec +# N-tile sweep: N=128 is a single N-tile; N=256/512 span 2 and 4 TileN blocks and +# exercise the round-6 per-N-tile PermuteN de-permute (the round-5 global riffle +# scrambled columns at N>=256). gN must divide N; TileN is 64 (decode) / 128 (MX). +# +# A single pinned N under TileN would leave the de-permute entirely unexercised, +# so this is coverage, not a knob: shrink it only with a replacement for that path. +_N_SWEEP = (128, 256, 512) + + def test_c4_fp8(out_dir: Path, gfx_arch: str) -> tuple[str, str]: # K=768 = 3*TileK(256): use num_loop=3 (TailNumber::Odd) for better coverage. # num_loop=2 works but exercises only the no-hot-loop/Even tail path; 3 gives # the no-hot-loop/Odd tail path and exercises the BQ scale prefetch more robustly. - M, N, K, gK, gN = 16, 64, 768, 128, 1 + M, K, gK, gN = 16, 768, 128, 1 cfg = default_fp8_config(quant_group_k=gK, quant_group_n=gN, gfx_arch=gfx_arch) - A_raw, A_dec, B_raw, B_dec, BQ = _make_fp8_inputs(M, N, K, gK, gN, "fp8", gfx_arch) - return _run_one("C4/fp8", cfg, M, N, K, - A_raw, A_dec, B_raw, B_dec, BQ, - out_dir, c_dtype=np.float16, gfx_arch=gfx_arch) + for N in _N_SWEEP: + A_raw, A_dec, B_raw, B_dec, BQ = _make_fp8_inputs(M, N, K, gK, gN, "fp8", + gfx_arch=gfx_arch) + status, detail = _run_one(f"C4/fp8/N{N}", cfg, M, N, K, + A_raw, A_dec, B_raw, B_dec, BQ, + out_dir, c_dtype=np.float16, gfx_arch=gfx_arch) + if status != PASS: + return status, detail + return PASS, f"C4/fp8: PASS for N in {_N_SWEEP}" def test_c4_bf8(out_dir: Path, gfx_arch: str) -> tuple[str, str]: # K=768 = 3*TileK(256): use num_loop=3 for the same reason as test_c4_fp8. - M, N, K, gK, gN = 16, 64, 768, 128, 1 + M, K, gK, gN = 16, 768, 128, 1 cfg = default_bf8_config(quant_group_k=gK, quant_group_n=gN, gfx_arch=gfx_arch) - A_raw, A_dec, B_raw, B_dec, BQ = _make_fp8_inputs(M, N, K, gK, gN, "bf8", gfx_arch) - return _run_one("C4/bf8", cfg, M, N, K, - A_raw, A_dec, B_raw, B_dec, BQ, - out_dir, c_dtype=np.float16, gfx_arch=gfx_arch) + for N in _N_SWEEP: + A_raw, A_dec, B_raw, B_dec, BQ = _make_fp8_inputs(M, N, K, gK, gN, "bf8", + gfx_arch=gfx_arch) + status, detail = _run_one(f"C4/bf8/N{N}", cfg, M, N, K, + A_raw, A_dec, B_raw, B_dec, BQ, + out_dir, c_dtype=np.float16, gfx_arch=gfx_arch) + if status != PASS: + return status, detail + return PASS, f"C4/bf8: PASS for N in {_N_SWEEP}" def test_h3_mx_bf16bf16(out_dir: Path, gfx_arch: str) -> tuple[str, str]: @@ -364,14 +394,18 @@ def test_h3_mx_bf16bf16(out_dir: Path, gfx_arch: str) -> tuple[str, str]: return SKIP, (f"H3/mx_bf16bf16: not supported on {gfx_arch} " f"(requires {'/'.join(MX_SUPPORTED_ARCHS)})") # K=256 = 2*TileK(128): MicroscaleCompV3 needs num_loop>=2 to avoid OOB second prefetch - M, N, K, gK, gN = 128, 128, 256, 32, 1 + M, K, gK, gN = 128, 256, 32, 1 cfg = default_mx_bf16bf16_config(quant_group_k=gK, quant_group_n=gN, gfx_arch=gfx_arch) - # C is bf16_t (2 bytes); use uint16 buffer + decode fn so size matches exactly. - A_raw, A_dec, B_raw, B_dec, BQ_e8m0, BQ_f32 = _make_bf16_inputs(M, N, K, gK, gN) - return _run_one("H3/mx_bf16bf16", cfg, M, N, K, - A_raw, A_dec, B_raw, B_dec, BQ_e8m0, - out_dir, c_dtype=np.uint16, c_decode_fn=_bf16_raw_to_f32, - BQ_ref=BQ_f32, gfx_arch=gfx_arch) + for N in _N_SWEEP: + # C is bf16_t (2 bytes); use uint16 buffer + decode fn so size matches exactly. + A_raw, A_dec, B_raw, B_dec, BQ_e8m0, BQ_f32 = _make_bf16_inputs(M, N, K, gK, gN) + status, detail = _run_one(f"H3/mx_bf16bf16/N{N}", cfg, M, N, K, + A_raw, A_dec, B_raw, B_dec, BQ_e8m0, + out_dir, c_dtype=np.uint16, c_decode_fn=_bf16_raw_to_f32, + BQ_ref=BQ_f32, gfx_arch=gfx_arch) + if status != PASS: + return status, detail + return PASS, f"H3/mx_bf16bf16: PASS for N in {_N_SWEEP}" def test_h3_mx_bf16bf8(out_dir: Path, gfx_arch: str) -> tuple[str, str]: @@ -379,17 +413,21 @@ def test_h3_mx_bf16bf8(out_dir: Path, gfx_arch: str) -> tuple[str, str]: return SKIP, (f"H3/mx_bf16bf8: not supported on {gfx_arch} " f"(requires {'/'.join(MX_SUPPORTED_ARCHS)})") # K=384 = 3*TileK(128): use num_loop=3 (TailNumber::Odd) for broader pipeline coverage. - M, N, K, gK, gN = 128, 128, 384, 128, 1 + M, K, gK, gN = 128, 384, 128, 1 cfg = default_mx_bf16bf8_config(quant_group_k=gK, quant_group_n=gN, gfx_arch=gfx_arch) - A_raw, A_dec, _, _, BQ_e8m0, BQ_f32 = _make_bf16_inputs(M, N, K, gK, gN) - # B is bf8: encode K*N float32 values as bf8 bytes - B_f32 = np.random.default_rng(43).uniform(-1.0, 1.0, (K, N)).astype(np.float32) - B_raw_bf8 = _encode_fp8(B_f32, "bf8", gfx_arch) - B_dec_bf8 = _decode_fp8(B_raw_bf8, "bf8", gfx_arch) - return _run_one("H3/mx_bf16bf8", cfg, M, N, K, - A_raw, A_dec, B_raw_bf8, B_dec_bf8, BQ_e8m0, - out_dir, c_dtype=np.uint16, c_decode_fn=_bf16_raw_to_f32, - BQ_ref=BQ_f32, gfx_arch=gfx_arch) + for N in _N_SWEEP: + A_raw, A_dec, _, _, BQ_e8m0, BQ_f32 = _make_bf16_inputs(M, N, K, gK, gN) + # B is bf8: encode K*N float32 values as bf8 bytes + B_f32 = np.random.default_rng(43).uniform(-1.0, 1.0, (K, N)).astype(np.float32) + B_raw_bf8 = _encode_fp8(B_f32, "bf8", gfx_arch) + B_dec_bf8 = _decode_fp8(B_raw_bf8, "bf8", gfx_arch) + status, detail = _run_one(f"H3/mx_bf16bf8/N{N}", cfg, M, N, K, + A_raw, A_dec, B_raw_bf8, B_dec_bf8, BQ_e8m0, + out_dir, c_dtype=np.uint16, c_decode_fn=_bf16_raw_to_f32, + BQ_ref=BQ_f32, gfx_arch=gfx_arch) + if status != PASS: + return status, detail + return PASS, f"H3/mx_bf16bf8: PASS for N in {_N_SWEEP}" def test_h3_mx_bf16fp4(out_dir: Path, gfx_arch: str) -> tuple[str, str]: @@ -398,23 +436,78 @@ def test_h3_mx_bf16fp4(out_dir: Path, gfx_arch: str) -> tuple[str, str]: f"(requires {'/'.join(MX_SUPPORTED_ARCHS)})") # pk_fp4: 2 fp4 values per byte → B buffer is K*N/2 bytes. # K=256 = 2*TileK(128): MicroscaleCompV3 needs num_loop>=2 to avoid OOB second prefetch. - M, N, K, gK, gN = 128, 128, 256, 32, 1 + M, K, gK, gN = 128, 256, 32, 1 cfg = default_mx_bf16fp4_config(quant_group_k=gK, quant_group_n=gN, gfx_arch=gfx_arch) - A_raw, A_dec, _, _, BQ_e8m0, BQ_f32 = _make_bf16_inputs(M, N, K, gK, gN) - rng = np.random.default_rng(44) - # rcr kernel reads pk_fp4 B COLUMN-MAJOR: consecutive K-elements share a byte - # (low nibble = k even, high nibble = k odd), per reference_mx_gemm_bquant's (k&1) - # branch. Generate logical (K,N) fp4 codes, pack column-major, LUT-decode reference. - codes = rng.integers(0, 16, size=(K, N), dtype=np.uint8) - B_f32_approx = _FP4_E2M1_LUT[codes] # reference values (K,N) - _flat = codes.flatten(order='F').astype(np.uint8) # col-major: idx = n*K + k - _lo = _flat[0::2] & 0x0F # k even -> low nibble - _hi = _flat[1::2] & 0x0F # k odd -> high nibble - B_raw = (_lo | (_hi << 4)).astype(np.uint8) - return _run_one("H3/mx_bf16fp4", cfg, M, N, K, - A_raw, A_dec, B_raw, B_f32_approx, BQ_e8m0, - out_dir, c_dtype=np.uint16, c_decode_fn=_bf16_raw_to_f32, - BQ_ref=BQ_f32, gfx_arch=gfx_arch) + for N in _N_SWEEP: + A_raw, A_dec, _, _, BQ_e8m0, BQ_f32 = _make_bf16_inputs(M, N, K, gK, gN) + rng = np.random.default_rng(44) + # rcr kernel reads pk_fp4 B COLUMN-MAJOR: consecutive K-elements share a byte + # (low nibble = k even, high nibble = k odd), per reference_mx_gemm_bquant's (k&1) + # branch. Generate logical (K,N) fp4 codes, pack column-major, LUT-decode reference. + codes = rng.integers(0, 16, size=(K, N), dtype=np.uint8) + B_f32_approx = _FP4_E2M1_LUT[codes] # reference values (K,N) + _flat = codes.flatten(order='F').astype(np.uint8) # col-major: idx = n*K + k + _lo = _flat[0::2] & 0x0F # k even -> low nibble + _hi = _flat[1::2] & 0x0F # k odd -> high nibble + B_raw = (_lo | (_hi << 4)).astype(np.uint8) + status, detail = _run_one(f"H3/mx_bf16fp4/N{N}", cfg, M, N, K, + A_raw, A_dec, B_raw, B_f32_approx, BQ_e8m0, + out_dir, c_dtype=np.uint16, c_decode_fn=_bf16_raw_to_f32, + BQ_ref=BQ_f32, gfx_arch=gfx_arch) + if status != PASS: + return status, detail + return PASS, f"H3/mx_bf16fp4: PASS for N in {_N_SWEEP}" + + +def test_c_i4(out_dir: Path, gfx_arch: str) -> tuple[str, str]: + """Round-6: fp8i4 / bf8i4 must be exact once BQ is encoded to the kernel's + QDataType (fp8/bf8, 1 byte). The round-5 float32 BQ produced NaN. Swept over + N to also exercise the per-N-tile de-permute. B is pk_int4 (2 per byte).""" + M, K, gK, gN = 16, 256, 128, 1 + results = [] + for variant, ctor, dtype in (("fp8i4", default_fp8i4_config, "fp8"), + ("bf8i4", default_bf8i4_config, "bf8")): + cfg = ctor(quant_group_k=gK, quant_group_n=gN, gfx_arch=gfx_arch) + for N in _N_SWEEP: + rng = np.random.default_rng(45) + A_f32 = rng.uniform(-2.0, 2.0, (M, K)).astype(np.float32) + A_raw = _encode_fp8(A_f32, dtype, gfx_arch) + A_dec = _decode_fp8(A_raw, dtype, gfx_arch) + # B is pk_int4: 2 K-elements per byte, packed COLUMN-MAJOR (rcr). + # + # Ground-truth nibble convention (Old-TE reference_gemm_quant + + # pk_int4.hpp with CK_TILE_USE_PK4_LAYOUT_SHUFFLE=1): + # * value(code) = code - 8 (code in 0..15 -> value in [-8, +7]), + # NOT the two's-complement (codes<8?codes:codes-16) round-5 used. + # * pk_int4_t_to_fp32x2_t returns {x_h, x_l} (SHUFFLE swap), and + # load_b picks (k&1) ? .hi : .lo, so for the byte covering + # (k even, k odd) of a fixed n: + # k even -> HIGH nibble; k odd -> LOW nibble. + # The device conversion consumes the same bytes after the + # UNCONDITIONAL permute_vectors_i4x4_b in the ctypes lib, so the + # host byte layout must match this reference exactly. + codes = rng.integers(0, 16, size=(K, N), dtype=np.uint8) # 0..15 + B_dec = codes.astype(np.float32) - 8.0 # value = code-8 + _flat = codes.flatten(order='F').astype(np.uint8) # col-major idx=n*K+k + _even = _flat[0::2] & 0x0F # k even -> HIGH nibble + _odd = _flat[1::2] & 0x0F # k odd -> LOW nibble + B_raw = (_odd | (_even << 4)).astype(np.uint8) + QK_B = math.ceil(K / gK) + QN_B = math.ceil(N / gN) + # BQ supplied as float32 -> the runner encodes it to fp8/bf8 (QDataType, + # arch-aware). The kernel therefore sees the fp8/bf8-rounded scale, so + # the CPU reference must use the same round-tripped value to stay fair. + BQ = rng.uniform(0.5, 2.0, (QK_B, QN_B)).astype(np.float32) + BQ_ref = _decode_fp8(_encode_fp8(BQ, dtype, gfx_arch), dtype, gfx_arch) + BQ_ref = BQ_ref.reshape(QK_B, QN_B).astype(np.float32) + status, detail = _run_one(f"C/{variant}/N{N}", cfg, M, N, K, + A_raw, A_dec, B_raw, B_dec, BQ, + out_dir, c_dtype=np.float16, + BQ_ref=BQ_ref, gfx_arch=gfx_arch) + if status != PASS: + return status, detail + results.append(variant) + return PASS, f"C/i4: PASS ({', '.join(results)}) for N in {_N_SWEEP}" # --------------------------------------------------------------------------- @@ -424,6 +517,7 @@ def test_h3_mx_bf16fp4(out_dir: Path, gfx_arch: str) -> tuple[str, str]: TESTS = [ ("C4/fp8", test_c4_fp8), ("C4/bf8", test_c4_bf8), + ("C/i4", test_c_i4), ("H3/mx_bf16bf16", test_h3_mx_bf16bf16), ("H3/mx_bf16bf8", test_h3_mx_bf16bf8), ("H3/mx_bf16fp4", test_h3_mx_bf16fp4), @@ -452,6 +546,14 @@ def main(): if gfx not in SUPPORTED_ARCHS: print(f"SKIP: BQuant is {'/'.join(SUPPORTED_ARCHS)}-only; detected {gfx}") return SKIP_EXIT + try: + import ml_dtypes # noqa: F401 + except ImportError: + # Gate here rather than letting _encode_fp8 raise: an ImportError inside a + # test function is swallowed by the loop's `except Exception` below and + # reported as FAIL, blaming the kernel for a missing host dependency. + print("SKIP: ml_dtypes not installed; BQuant fp8/bf8 encoding unavailable") + return SKIP_EXIT out_dir = args.output_dir or Path(tempfile.mkdtemp(prefix="bquant_gpu_test_")) log.info("Kernel output dir: %s", out_dir) diff --git a/projects/composablekernel/dispatcher/tests/test_contraction_multi_abd_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_contraction_multi_abd_gpu_correctness.py index 5053c4bfff59..3a4f99c52a34 100644 --- a/projects/composablekernel/dispatcher/tests/test_contraction_multi_abd_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_contraction_multi_abd_gpu_correctness.py @@ -61,6 +61,10 @@ PASS = "PASS" FAIL = "FAIL" +# ctest SKIP_RETURN_CODE: main() returns this when the box cannot run the test +# at all, so the lane reports Skipped rather than a vacuous Passed. +SKIP_EXIT = 77 + # Must match the kernel's default tile (256x256x64) exactly: the default config # pads nothing, so M and N have to be whole multiples of the tile or # IsSupportedArguments rejects the launch. @@ -189,7 +193,7 @@ def main() -> int: if not _has_gpu(): print("SKIP: no supported GPU or hipcc detected; " "contraction_multi_abd GPU test skipped") - return 0 + return SKIP_EXIT gfx = _validate_arch(args.gfx) if args.gfx else _detect_gpu_arch() log.info("Running contraction_multi_abd GPU correctness on %s", gfx) diff --git a/projects/composablekernel/dispatcher/tests/test_dispatcher_common.py b/projects/composablekernel/dispatcher/tests/test_dispatcher_common.py index 2c0fc8307cdb..c7407f00b8e1 100644 --- a/projects/composablekernel/dispatcher/tests/test_dispatcher_common.py +++ b/projects/composablekernel/dispatcher/tests/test_dispatcher_common.py @@ -39,6 +39,10 @@ print_error, print_info, cleanup_generated_kernels, + normalize_arch, + fp8_uses_ocp, + ocp_arch_defines, + arch_feature_defines, ) @@ -239,5 +243,143 @@ def test_cleanup_nonexistent_dir_no_error(self): cleanup_generated_kernels(Path("/tmp/nonexistent_ck_test_dir_12345")) +class TestNormalizeArch(unittest.TestCase): + """Tests for normalize_arch -- the front door for every arch predicate.""" + + def test_bare_names_pass_through(self): + for arch in ("gfx90a", "gfx942", "gfx950", "gfx1200", "gfx1250"): + with self.subTest(arch=arch): + self.assertEqual(normalize_arch(arch), arch) + + def test_feature_suffix_is_stripped(self): + # What rocminfo and the HSA runtime actually hand back. + self.assertEqual(normalize_arch("gfx950:sramecc+:xnack-"), "gfx950") + self.assertEqual(normalize_arch("gfx942:sramecc+:xnack-"), "gfx942") + self.assertEqual(normalize_arch("gfx90a:xnack+"), "gfx90a") + + def test_offload_target_prefix_is_stripped(self): + # What --offload-arch round-trips through in some toolchain paths. This + # is the form the old startswith() test missed outright. + self.assertEqual(normalize_arch("amdgcn-amd-amdhsa--gfx950"), "gfx950") + self.assertEqual( + normalize_arch("amdgcn-amd-amdhsa--gfx950:sramecc+:xnack-"), "gfx950" + ) + + def test_case_and_whitespace_are_normalized(self): + self.assertEqual(normalize_arch(" GFX950 "), "gfx950") + + def test_none_and_empty_are_empty(self): + self.assertEqual(normalize_arch(None), "") + self.assertEqual(normalize_arch(""), "") + + +class TestFp8ArchPredicates(unittest.TestCase): + """fp8_uses_ocp and the define lists derived from it. + + A miss here is silent: the -DCK_USE_OCP_FP8 pair simply does not get passed, + the host pass of config.hpp falls back to FNUZ, and the kernel builds and + runs while disagreeing with its own reference on what the bytes mean. + """ + + _OCP = ("gfx950", "gfx1200", "gfx1201", "gfx1250") + _FNUZ = ("gfx90a", "gfx908", "gfx942", "gfx1100", "gfx1030") + + def test_ocp_arches(self): + for arch in self._OCP: + with self.subTest(arch=arch): + self.assertTrue(fp8_uses_ocp(arch)) + + def test_fnuz_arches(self): + for arch in self._FNUZ: + with self.subTest(arch=arch): + self.assertFalse(fp8_uses_ocp(arch)) + + def test_full_target_triples_agree_with_bare_names(self): + for arch in self._OCP + self._FNUZ: + triple = f"{arch}:sramecc+:xnack-" + with self.subTest(arch=triple): + self.assertEqual(fp8_uses_ocp(triple), fp8_uses_ocp(arch)) + + def test_offload_target_form_agrees_with_bare_name(self): + for arch in self._OCP + self._FNUZ: + target = f"amdgcn-amd-amdhsa--{arch}" + with self.subTest(arch=target): + self.assertEqual(fp8_uses_ocp(target), fp8_uses_ocp(arch)) + + def test_unknown_and_none_default_to_fnuz(self): + # FNUZ is the compiler default for both passes, so it is also the safe + # answer for an arch we do not recognize. + for arch in (None, "", "not-a-gpu"): + with self.subTest(arch=arch): + self.assertFalse(fp8_uses_ocp(arch)) + + def test_ocp_defines_are_emitted_as_a_pair(self): + # Passing one without the other splits ck/ from ck_tile/. + self.assertEqual( + ocp_arch_defines("gfx950"), + ["-DCK_USE_OCP_FP8", "-DCK_TILE_USE_OCP_FP8"], + ) + self.assertEqual(ocp_arch_defines("gfx942"), []) + + +class TestArchFeatureDefines(unittest.TestCase): + """arch_feature_defines mirrors the top-level CMakeLists.txt arch block.""" + + def test_gfx942_gets_no_feature_defines(self): + self.assertEqual(arch_feature_defines("gfx942"), []) + + def test_gfx950_gets_ocp_plus_mx(self): + self.assertEqual( + arch_feature_defines("gfx950"), + [ + "-DCK_USE_OCP_FP8", + "-DCK_TILE_USE_OCP_FP8", + "-DCK_USE_NATIVE_MX_SUPPORT", + "-DCK_GFX950_SUPPORT", + ], + ) + + def test_gfx12_enables_wmma(self): + # CK_TILE_USE_WMMA must be passed explicitly: it selects the outermost + # branch of get_k_warp_tile(), and leaving it undefined reads as 0 -- + # the MFMA branch, which no gfx12 part has. + defines = arch_feature_defines("gfx1200") + self.assertIn("-DCK_TILE_USE_WMMA=1", defines) + self.assertIn("-DCK_GFX12_SUPPORT", defines) + self.assertIn("-DCK_USE_OCP_FP8", defines) + self.assertNotIn("-DCK_GFX950_SUPPORT", defines) + + def test_gfx1250_gets_its_own_defines_on_top_of_gfx12(self): + defines = arch_feature_defines("gfx1250") + for expected in ( + "-DCK_TILE_USE_WMMA=1", + "-DCK_GFX12_SUPPORT", + "-DCK_USE_GFX1250", + "-DCK_USE_NATIVE_MX_SUPPORT", + "-DCK_GFX1250_SUPPORT", + ): + with self.subTest(define=expected): + self.assertIn(expected, defines) + + def test_gfx950_and_gfx12_defines_are_mutually_exclusive(self): + # get_k_warp_tile() branches on these; both at once is not a state the + # header is written for. + for arch in ("gfx950", "gfx1250"): + with self.subTest(arch=arch): + defines = arch_feature_defines(arch) + self.assertNotEqual( + "-DCK_GFX950_SUPPORT" in defines, + "-DCK_TILE_USE_WMMA=1" in defines, + ) + + def test_target_triple_gets_the_same_defines_as_the_bare_name(self): + for arch in ("gfx942", "gfx950", "gfx1250"): + with self.subTest(arch=arch): + self.assertEqual( + arch_feature_defines(f"{arch}:sramecc+:xnack-"), + arch_feature_defines(arch), + ) + + if __name__ == "__main__": unittest.main() diff --git a/projects/composablekernel/dispatcher/tests/test_gemm_rowcolquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_gemm_rowcolquant_gpu_correctness.py index 5d86bbafffc6..dbd79155c90c 100644 --- a/projects/composablekernel/dispatcher/tests/test_gemm_rowcolquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_gemm_rowcolquant_gpu_correctness.py @@ -34,11 +34,20 @@ Run: python3 -m pytest test_gemm_rowcolquant_gpu_correctness.py -v python3 test_gemm_rowcolquant_gpu_correctness.py # standalone + python3 test_gemm_rowcolquant_gpu_correctness.py --gfx gfx942 + +The standalone path is what ctest and the Jenkins lane drive; it exits 77 (see +SKIP_EXIT) when the box cannot run the test at all, so an unrunnable runner is +recorded as Skipped rather than as a vacuous Passed or a spurious Failed. """ +import argparse +import logging +import subprocess import sys import tempfile from pathlib import Path +from typing import Optional import numpy as np import pytest @@ -80,11 +89,43 @@ def _find_python_dir() -> Path: ) +log = logging.getLogger(__name__) + +PASS = "PASS" +FAIL = "FAIL" +SKIP_EXIT = 77 # ctest SKIP_RETURN_CODE; see main() + +# RowColQuant is a block-scale fp8/bf8 op: it needs native fp8 MFMA. A GPU +# outside this set clears conftest's gpu_available() probe and then fails at +# hipcc, which reads as a correctness FAIL rather than "this box cannot run it". +# Matches SUPPORTED_ARCHS in the grouped tensorquant/rowcolquant GPU tests. +_SUPPORTED_ARCHS = ("gfx942", "gfx950") + + def _have_ml_dtypes() -> bool: # rowcolquant's fp8 encoders come from ml_dtypes via its own utils. return u.fp8_encoding_available() +def _has_hipcc() -> bool: + try: + subprocess.run(["hipcc", "--version"], capture_output=True, timeout=10) + return True + except Exception: + return False + + +def _safe_detect_arch() -> Optional[str]: + """_detect_gpu_arch() raises on absence/unsupported; the gates want None.""" + try: + return _detect_gpu_arch() + except Exception: + return None + + +_GFX_ARCH = _safe_detect_arch() + + # ============================================================================= # NumPy reference: C[m,n] = AQ[m] * BQ[n] * sum_k A[m,k] B[k,n] # ============================================================================= @@ -102,8 +143,11 @@ def reference_rowcolquant_gemm(A_dec, B_dec, AQ, BQ) -> np.ndarray: # ============================================================================= -def _run_case(dtype: str, M: int, N: int, K: int, out_dir: Path): - arch = _detect_gpu_arch() +def _run_case(dtype: str, M: int, N: int, K: int, out_dir: Path, + arch: Optional[str] = None): + # main() resolves the arch once (honouring --gfx) and passes it down; the + # pytest entry points leave it None and take the detected one. + arch = arch or _detect_gpu_arch() make_cfg = default_fp8_config if dtype == "fp8" else default_bf8_config config = make_cfg(gfx_arch=arch) @@ -149,10 +193,19 @@ def _run(so_path): _SKIP_NO_GPU = pytest.mark.skipif(not _have_gpu(), reason="no ROCm GPU detected") _SKIP_NO_MLD = pytest.mark.skipif(not _have_ml_dtypes(), reason="ml_dtypes not installed") +# gpu_available() only says "a ROCm GPU is present". Without this second gate a +# gfx90a box runs the fp8 build and reports a red FAIL for hardware that was +# never in scope. +_SKIP_BAD_ARCH = pytest.mark.skipif( + _GFX_ARCH not in _SUPPORTED_ARCHS, + reason=f"RowColQuant needs native fp8 ({'/'.join(_SUPPORTED_ARCHS)}); " + f"detected {_GFX_ARCH}", +) @_SKIP_NO_GPU @_SKIP_NO_MLD +@_SKIP_BAD_ARCH @pytest.mark.parametrize("dtype", ["fp8", "bf8"]) def test_rowcolquant_gpu_matches_reference(dtype, tmp_path): max_rel, _ = _run_case(dtype, M=256, N=256, K=512, out_dir=tmp_path) @@ -161,22 +214,91 @@ def test_rowcolquant_gpu_matches_reference(dtype, tmp_path): @_SKIP_NO_GPU @_SKIP_NO_MLD +@_SKIP_BAD_ARCH def test_rowcolquant_gpu_not_all_zeros(tmp_path): _run_case("fp8", M=256, N=256, K=512, out_dir=tmp_path) -if __name__ == "__main__": - if not _have_gpu(): - print("SKIP: no GPU"); raise SystemExit(0) +# ============================================================================= +# Standalone entry point (the one ctest and Jenkins drive) +# ============================================================================= + + +def _case(dtype: str): + """Build a never-raising case fn returning (status, detail). + + The TESTS-loop convention is that a case reports its own verdict; main()'s + ``except Exception`` is a backstop for genuine bugs, not the normal path. + """ + def run(out_dir: Path, arch: str): + max_rel, time_ms = _run_case(dtype, 256, 256, 512, out_dir, arch) + if max_rel > 0.05: + return FAIL, f"rowcolquant {dtype}: max_rel={max_rel:.4f} > tol=0.05" + return PASS, (f"rowcolquant {dtype}: max_rel={max_rel:.4f} " + f"time_ms={time_ms:.3f}") + return run + + +TESTS = [(f"rowcolquant_{dt}", _case(dt)) for dt in ("fp8", "bf8")] + + +def main() -> int: + parser = argparse.ArgumentParser( + description="RowColQuant (non-grouped) GPU correctness tests") + # No hardcoded default: an explicit --gfx is an override the caller is + # trusted on, but falling back to a fixed arch would make a CPU-only or + # gfx90a box compile for gfx950 and crash instead of skipping. + parser.add_argument("--gfx", default=None, + help=f"GPU arch override (default: auto-detect; " + f"{'/'.join(_SUPPORTED_ARCHS)} only)") + parser.add_argument("-v", "--verbose", action="store_true") + parser.add_argument("--output-dir", type=Path, default=None) + args = parser.parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(levelname)s: %(message)s", + ) + + # The pytest path gates via the skipif marks above; this standalone path is + # what CI drives, so it needs the same gates spelled out. SKIP_EXIT keeps a + # clean skip distinguishable from a failure -- ctest maps it via + # SKIP_RETURN_CODE and the Jenkins lane via its run_ok helper. + gfx = args.gfx or _GFX_ARCH + if not gfx: + print("SKIP: no supported GPU detected (rocm_agent_enumerator)") + return SKIP_EXIT + if gfx not in _SUPPORTED_ARCHS: + print(f"SKIP: RowColQuant needs native fp8 " + f"({'/'.join(_SUPPORTED_ARCHS)}); detected {gfx}") + return SKIP_EXIT + if not _has_hipcc(): + print("SKIP: hipcc not found in PATH; cannot JIT-compile kernels") + return SKIP_EXIT if not _have_ml_dtypes(): - print("SKIP: ml_dtypes not installed"); raise SystemExit(0) - d = Path(tempfile.mkdtemp(prefix="rowcolquant_gputest_")) - ok = True - for dt in ("fp8", "bf8"): + print("SKIP: ml_dtypes not installed; fp8/bf8 encoding unavailable") + return SKIP_EXIT + + out_dir = args.output_dir or Path(tempfile.mkdtemp(prefix="rowcolquant_gputest_")) + log.info("Kernel output dir: %s", out_dir) + + results = [] + for name, fn in TESTS: + log.info("--- Running %s ---", name) try: - mr, t = _run_case(dt, 256, 256, 512, d) - print(f"PASS rowcolquant {dt}: max_rel={mr:.4f} time_ms={t:.3f}") - except Exception as e: - ok = False - print(f"FAIL rowcolquant {dt}: {e}") - raise SystemExit(0 if ok else 1) + status, detail = fn(out_dir, gfx) + except Exception as exc: # noqa: BLE001 + status, detail = FAIL, f"{name}: exception: {exc}" + results.append((status, detail)) + log.info("[%s] %s", status, detail) + + print("\n=== Summary ===") + passed = sum(1 for s, _ in results if s == PASS) + for status, detail in results: + print(f" [{status:4s}] {detail}") + print(f"\n{passed}/{len(results)} passed") + return 0 if passed == len(results) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py b/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py index 12fd6057c3bc..0210e72c17e6 100644 --- a/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py +++ b/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py @@ -6,9 +6,10 @@ Dispatcher GEMM search space runner (all bridged GEMM variants). Enumerates the GemmKernelConfig search space by calling expand_sweep with the -tile_engine default_ci_config.json across all (dtype, layout) strata, samples a -budget-limited subset with a daily rotating seed, compiles each config via the -bridge, runs on GPU, and reports correctness + TFLOPS. +variant's tile_engine default_ci_config.json (multi_d and multi_abd have their +own, carrying the fused-epilogue block) across all (dtype, layout) strata, +samples a budget-limited subset with a daily rotating seed, compiles each config +via the bridge, runs on GPU, and reports correctness + TFLOPS. One runner covers every variant expand_sweep understands -- standard, grouped, multi_d, multi_abd and stream_k -- because they share the same enumerate/sample/ @@ -27,8 +28,8 @@ python3 test_gemm_search_space.py --arch gfx942 --budget 500 python3 test_gemm_search_space.py --variant grouped --groups 4 python3 test_gemm_search_space.py --variant multi_abd # fp16/rcrr only - python3 test_gemm_search_space.py --variant multi_d # PassThrough epilogue - python3 test_gemm_search_space.py --variant multi_d --elementwise-op MultiDAdd + python3 test_gemm_search_space.py --variant multi_d # sweeps MultiDAdd/Multiply + python3 test_gemm_search_space.py --variant multi_d --elementwise-op PassThrough python3 test_gemm_search_space.py --budget 0 # full space, no cap python3 test_gemm_search_space.py --budget 500 --seed 42 --json results.json """ @@ -63,14 +64,36 @@ setup_multiple_gemm_dispatchers, ) -_CI_CONFIG = ( - _DISPATCHER.parent / "tile_engine/ops/gemm/configs/default_ci_config.json" -) +_TE_GEMM = _DISPATCHER.parent / "tile_engine/ops/gemm" +_CI_CONFIG = _TE_GEMM / "configs/default_ci_config.json" + +# multi_d and multi_abd have their own CI configs carrying the +# ``multi_d_config`` / ``multi_abd_config`` blocks that describe the fused +# epilogue (which element-wise op, how many D tensors). The generic config has +# neither, so pointing every variant at it left expand_sweep with nothing to +# enumerate and the fusion untested. gemm_multi_d_full_benchmark.py already +# reads these same files. +_VARIANT_CI_CONFIG = { + "multi_d": _TE_GEMM / "gemm_multi_d/configs/default_ci_config.json", + "multi_abd": _TE_GEMM / "gemm_multi_abd/configs/default_ci_config.json", +} + + +def _ci_config_for(variant: str) -> Path: + return _VARIANT_CI_CONFIG.get(variant, _CI_CONFIG) _DTYPES = ["fp16", "bf16", "fp8", "bf8"] _LAYOUTS = ["rcr", "rrr", "crr", "ccr"] _RTOL = 2e-2 _SKIP = 77 +# The arches whose kernels this sweep can actually build and run, matching +# SUPPORTED_ARCHS in the aquant/abquant GPU tests. The no-GPU gate below is not +# sufficient on its own: a box with an *unsupported* GPU (gfx90a, say) passes it, +# then fails every sampled config at the hipcc step and reports FAIL. That is the +# same false signal the exit-77 convention exists to remove, pointing the other +# way -- so gate on the resolved arch, forced or detected, exactly as bquant does. +_SUPPORTED_ARCHS = ("gfx942", "gfx950") + # Per-variant defaults for --dtypes / --layouts. multi_abd and multi_d are # fp16-only end-to-end (codegen, ctypes lib and runner all assume fp16) and take # the 4-char (A,B,C/E,D) layout code rather than the 3-char one. Only the A and @@ -236,9 +259,12 @@ def _verify(variant: str, cfg, runner, result, ops: _Operands) -> float: # multi_d's fused epilogue op lives in `elementwise_op`; the similarly # named `cde_elementwise_op` is the multi_abd field and stays at its # PassThrough default here, so reading it would reference the wrong op. - # Read it rather than hard-coding --elementwise-op: under the default - # PassThrough the epilogue is E = C and _cde_reference drops the D - # tensors, which is only correct because both sides agree on the op. + # Read it per-config rather than from --elementwise-op: the sweep now + # enumerates MultiDAdd and MultiDMultiply together, so a single pinned + # op would mis-reference half the configs. + # + # nd comes from the kernel, not the config, and the D pool is sized to + # the largest num_d in the sample -- num_d is itself swept over {1, 2}. nd = runner.num_d_tensors acc = _quantize(ops.A, dtype) @ _quantize(ops.B, dtype) ref = _cde_reference(cfg.elementwise_op, acc, ops.Ds[:nd]) @@ -263,6 +289,11 @@ def run(args) -> int: print("SKIP: no GPU detected and --arch not given") return _SKIP + if arch not in _SUPPORTED_ARCHS: + print(f"SKIP: search-space sweep is {'/'.join(_SUPPORTED_ARCHS)}-only; " + f"got {arch}") + return _SKIP + variant = args.variant def_dtypes, def_layouts = _VARIANT_DEFAULTS[variant] dtypes = [d.strip() for d in args.dtypes.split(",")] if args.dtypes else def_dtypes @@ -278,16 +309,20 @@ def run(args) -> int: print("\nEnumerating search space...") t0 = time.time() strata = {} + ci_config = _ci_config_for(variant) for dtype in dtypes: for layout in layouts: - configs = expand_sweep(str(_CI_CONFIG), arch=arch, dtype=dtype, + configs = expand_sweep(str(ci_config), arch=arch, dtype=dtype, layout=layout, variant=variant) - if variant == "multi_d": - # default_ci_config.json carries no multi_d_config, so expand_sweep - # falls back to its own MultiDAdd default. Pin the epilogue op here - # instead: the config dataclass derives .name (and the codegen's - # elementwise_ops list) from this field, so replacing it is enough - # to select a different kernel -- no separate JSON is needed. + if variant == "multi_d" and args.elementwise_op is not None: + # Only when explicitly asked for. The config dataclass derives + # .name (and the codegen's elementwise_ops list) from this field, + # so replacing it is enough to select a different kernel -- which + # makes `--elementwise-op PassThrough` a usable A/B experiment. + # It is not a usable default: PassThrough takes Ds as an unnamed + # parameter pack device-side and _cde_reference returns acc + # host-side, so both sides compute A@B and a kernel that never + # loads D still passes. configs = [replace(c, elementwise_op=args.elementwise_op) for c in configs] strata[f"{dtype}/{layout}"] = configs @@ -414,10 +449,13 @@ def main() -> int: "rcrr for multi_d/multi_abd)") p.add_argument("--groups", type=int, default=4, help="Sub-problem count for --variant grouped (default: 4)") - p.add_argument("--elementwise-op", default="PassThrough", + p.add_argument("--elementwise-op", default=None, choices=("PassThrough", "MultiDAdd", "MultiDMultiply"), - help="Epilogue op for --variant multi_d (default: PassThrough). " - "Ignored by every other variant.") + help="Pin the epilogue op for --variant multi_d, overriding " + "the sweep (default: sweep MultiDAdd/MultiDMultiply from " + "the config). PassThrough discards the D tensors on both " + "the device and reference sides, so it verifies plain " + "A@B only. Ignored by every other variant.") p.add_argument("--budget", type=int, default=500, help="Max configs to run; 0 = no cap (default: 500)") p.add_argument("--seed", type=int, default=None, diff --git a/projects/composablekernel/dispatcher/tests/test_gemm_utils.py b/projects/composablekernel/dispatcher/tests/test_gemm_utils.py index 7533381284d0..d0a6c28b75e3 100644 --- a/projects/composablekernel/dispatcher/tests/test_gemm_utils.py +++ b/projects/composablekernel/dispatcher/tests/test_gemm_utils.py @@ -255,6 +255,79 @@ def test_numpy_dtype_follows_the_requested_format(self): np.dtype(ml_dtypes.float8_e5m2fnuz)) +class TestFp8EncodeMatchesMlDtypes(unittest.TestCase): + """_fp8_encode vs ml_dtypes: identical in range, divergent on overflow. + + _fp8_encode exists because ml_dtypes cannot express CK's saturating convert, + not because the rounding differs -- so everything except overflow has to + agree exactly, and the one place it does not has to be pinned as deliberate. + """ + + _CASES = ( + (4, 3, True, "float8_e4m3fn"), + (5, 2, True, "float8_e5m2"), + (4, 3, False, "float8_e4m3fnuz"), + (5, 2, False, "float8_e5m2fnuz"), + ) + + @staticmethod + def _finite_magnitudes(table): + return np.unique(table[np.isfinite(table)].astype(np.float64)) + + def test_midpoints_round_half_to_even(self): + # The regression this guards: ties were broken toward the smaller + # magnitude, which disagreed with the hardware convert (and ml_dtypes) + # on roughly half of all representable midpoints -- 126 of 252 for + # e4m3. Every one of those is a silently low reference value. + ml_dtypes = _require_ml_dtypes() + for exp_bits, mant_bits, use_ocp, ml_name in self._CASES: + with self.subTest(bits=(exp_bits, mant_bits), ocp=use_ocp): + table = _fp8_decode_table(exp_bits, mant_bits, use_ocp) + mags = self._finite_magnitudes(table) + mids = ((mags[:-1] + mags[1:]) / 2.0).astype(np.float32) + ours = table[_fp8_encode(mids, exp_bits, mant_bits, + use_ocp=use_ocp).astype(np.intp)] + theirs = mids.astype(getattr(ml_dtypes, ml_name)) + np.testing.assert_array_equal(ours.astype(np.float64), + theirs.astype(np.float64)) + + def test_agrees_across_the_whole_finite_range(self): + ml_dtypes = _require_ml_dtypes() + for exp_bits, mant_bits, use_ocp, ml_name in self._CASES: + with self.subTest(bits=(exp_bits, mant_bits), ocp=use_ocp): + table = _fp8_decode_table(exp_bits, mant_bits, use_ocp) + mags = self._finite_magnitudes(table) + max_finite = float(mags[-1]) + vals = np.unique(np.concatenate([ + mags, + -mags, + np.linspace(-max_finite, max_finite, 4001), + ])) + vals = vals[np.abs(vals) <= max_finite].astype(np.float32) + ours = table[_fp8_encode(vals, exp_bits, mant_bits, + use_ocp=use_ocp).astype(np.intp)] + theirs = vals.astype(getattr(ml_dtypes, ml_name)) + np.testing.assert_array_equal(ours.astype(np.float64), + theirs.astype(np.float64)) + + def test_overflow_saturates_where_ml_dtypes_does_not(self): + # Intended divergence, not a gap: CK's convert clamps, so the host + # reference must clamp too. Deferring to ml_dtypes here would poison the + # reference with NaN/Inf on any input the kernel merely saturates. + ml_dtypes = _require_ml_dtypes() + big = np.array([1e30, -1e30], dtype=np.float32) + for exp_bits, mant_bits, use_ocp, ml_name in self._CASES: + with self.subTest(bits=(exp_bits, mant_bits), ocp=use_ocp): + table = _fp8_decode_table(exp_bits, mant_bits, use_ocp) + max_finite = float(self._finite_magnitudes(table)[-1]) + ours = table[_fp8_encode(big, exp_bits, mant_bits, + use_ocp=use_ocp).astype(np.intp)] + self.assertEqual(float(ours[0]), max_finite) + self.assertEqual(float(ours[1]), -max_finite) + theirs = big.astype(getattr(ml_dtypes, ml_name)).astype(np.float64) + self.assertFalse(np.all(np.isfinite(theirs))) + + class TestOutputDtype(unittest.TestCase): """Output (C) element dtype must mirror the codegen's get_output_dtype.""" @@ -435,9 +508,15 @@ def test_ocp_fp8_roundtrip_exact_values(self): def test_ocp_differs_from_fnuz(self): # OCP and FNUZ are distinct encodings; the byte patterns must not match # for a generic value (guards against silently reusing the FNUZ codec). + # + # use_ocp=False is required, not incidental: _fp32_to_fp8_u8 defaults to + # autodetecting the local GPU, so on an OCP arch (gfx950/gfx12) both + # sides would encode OCP and the assertion below would fail for a reason + # unrelated to the codecs being distinct. x = np.array([[0.1, 0.3, 1.5]], dtype=np.float32) self.assertFalse( - np.array_equal(_fp32_to_fp8_ocp_u8(x), _fp32_to_fp8_u8(x)) + np.array_equal(_fp32_to_fp8_ocp_u8(x), + _fp32_to_fp8_u8(x, use_ocp=False)) ) def test_ocp_bf8_shape(self): diff --git a/projects/composablekernel/dispatcher/tests/test_multi_d_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_multi_d_gpu_correctness.py index 0913173471b2..cf168e1d2b03 100644 --- a/projects/composablekernel/dispatcher/tests/test_multi_d_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_multi_d_gpu_correctness.py @@ -16,16 +16,20 @@ The multi_d TE op is fp16-only, so only fp16 is exercised, but all four A/B layout combinations are (rcrr / rrrr / ccrr / crrr). -Runs green on gfx942 (MI300X). SKIPs cleanly with no GPU / hipcc / static lib. +Runs green on gfx942 (MI300X). Exits 77 (= ctest "Skipped") with no GPU / +static lib / hipcc, rather than reporting a vacuous PASS. Run: - python3 -m pytest tests/test_multi_d_gpu_correctness.py -v python3 tests/test_multi_d_gpu_correctness.py + python3 tests/test_multi_d_gpu_correctness.py -v # verbose hipcc output + python3 tests/test_multi_d_gpu_correctness.py --gfx gfx942 """ +import argparse +import logging import shutil import sys -import unittest +from functools import partial from pathlib import Path import numpy as np @@ -41,8 +45,19 @@ setup_multiple_gemm_dispatchers, ) +log = logging.getLogger(__name__) + _TOL = 1e-2 # fp16 GEMM + fp16 D-fuse precision band +PASS = "PASS" +FAIL = "FAIL" +SKIP = "SKIP" + +# ctest reports this as "skipped" rather than passed. Without it, a CPU-only +# runner would report every multi_d case as PASSED -- `-m pytest` exits 0 when +# every test in a file skips -- hiding a missing GPU behind a green result. +SKIP_EXIT = 77 + # 4-char (A, B, C/E, D) layout codes, matching _VARIANT_DEFAULTS["multi_d"] in # test_gemm_search_space.py. C and E are row-major throughout: the TE multi_d # epilogue writes row-major, so only the A and B chars actually vary. @@ -52,8 +67,8 @@ def _detect_arch(): # subprocess.run with a timeout (rocminfo can hang on misconfigured ROCm - # installs and would otherwise stall test discovery). Validate the parsed - # token is a real gfx string before returning it. Mirrors + # installs and would otherwise stall the run). Validate the parsed token is + # a real gfx string before returning it. Mirrors # dispatcher/python/ctypes_utils.py:detect_gpu_arch. import subprocess try: @@ -87,94 +102,133 @@ def _max_rel_err(got: np.ndarray, ref: np.ndarray) -> float: return float(np.max(np.abs(g - r) / den)) -class TestMultiDGemmGpu(unittest.TestCase): - ARCH = _detect_arch() - - def setUp(self): - if self.ARCH is None: - self.skipTest("no GPU / rocminfo not available") - if not _static_lib_present(): - self.skipTest( - "dispatcher static lib (libck_tile_dispatcher.a) not built; " - "multi_d is registry-routed and needs it" - ) - if shutil.which("hipcc") is None and not Path("/opt/rocm/bin/hipcc").exists(): - self.skipTest("hipcc not found") - - def test_fp16_multid_add(self): - """E = A@B + D0 + D1 across all four A/B layout combinations. - - The runner reads the layout back out of the kernel name and transposes - the host operands itself, so A and B are handed over as logical (M, K) - and (K, N) in every case and the reference below is layout-independent. - Each layout is a separate build, hence subTest rather than four methods: - a failure names the layout without hiding the other three. - """ - for layout in _LAYOUTS: - with self.subTest(layout=layout): - self._check_layout(layout) - - def _check_layout(self, layout: str): - num_d = 2 - la, lb, lc, ld = (_LAYOUT_WORD[c] for c in layout) - cfg = GemmKernelConfig( - dtype_a="fp16", dtype_b="fp16", dtype_c="fp16", dtype_acc="fp32", - layout_a=la, layout_b=lb, layout_c=lc, - tile_m=128, tile_n=128, tile_k=32, - wave_m=2, wave_n=2, wave_k=1, - warp_tile_m=32, warp_tile_n=32, warp_tile_k=16, - pipeline="compv4", scheduler="intrawave", epilogue="cshuffle", - pad_m=True, pad_n=True, pad_k=True, persistent=False, - variant="multi_d", elementwise_op="MultiDAdd", - num_d_tensors=num_d, d_layout=ld, - gfx_arch=self.ARCH, - ) - so_paths = setup_multiple_gemm_dispatchers([cfg], verbose=False) - so = so_paths[0] - if so is None: - self.fail(f"multi_d fp16 kernel failed to build (layout={layout})") - - M, N, K = 512, 512, 512 - problem = MultiDGemmProblem(M=M, N=N, K=K, num_d=num_d) - - rng = np.random.default_rng(11) - A = rng.uniform(-1.0, 1.0, (M, K)).astype(np.float32) - B = rng.uniform(-1.0, 1.0, (K, N)).astype(np.float32) - Ds = [rng.uniform(-1.0, 1.0, (M, N)).astype(np.float32) - for _ in range(num_d)] - - runner = GpuMultiDGemmRunner(so) - self.assertEqual(runner.num_d_tensors, num_d, - "kernel D-count mismatch") - result = runner.run(A.astype(np.float16), B.astype(np.float16), - [d.astype(np.float16) for d in Ds], problem) - self.assertEqual(result.status, 0, - f"multi_d run status={result.status} (layout={layout})") - self.assertGreater(result.time_ms, 0.0, - f"multi_d time_ms not positive (layout={layout})") - - # Reference at fp16 input precision, accumulated + fused in fp32: - # E = A@B + D0 + D1 (MultiDAdd). - A_q = A.astype(np.float16).astype(np.float32) - B_q = B.astype(np.float16).astype(np.float32) - E_ref = A_q @ B_q - for d in Ds: - E_ref = E_ref + d.astype(np.float16).astype(np.float32) - - E_got = np.asarray(result.output).astype(np.float32) - self.assertFalse(np.all(E_got == 0.0), - f"multi_d output all-zero (layout={layout})") - self.assertTrue(np.all(np.isfinite(E_got)), - f"multi_d output NaN/Inf (layout={layout})") - mre = _max_rel_err(E_got, E_ref) - self.assertLessEqual( - mre, _TOL, - f"multi_d fp16 layout={layout} max_rel={mre:.4f} > tol={_TOL} " - f"(M={M} N={N} K={K})", - ) - print(f"[multi_d/fp16/{layout}] max_rel={mre:.4e}, " - f"time_ms={result.time_ms:.3f}") +def _check_layout(layout: str, gfx_arch: str): + """E = A@B + D0 + D1 for one A/B layout combination. + + The runner reads the layout back out of the kernel name and transposes the + host operands itself, so A and B are handed over as logical (M, K) and + (K, N) in every case and the reference below is layout-independent. + + Returns (status, detail) and never raises: each layout is a separate hipcc + build, and a failure in one must name its layout without hiding the other + three. + """ + num_d = 2 + la, lb, lc, ld = (_LAYOUT_WORD[c] for c in layout) + cfg = GemmKernelConfig( + dtype_a="fp16", dtype_b="fp16", dtype_c="fp16", dtype_acc="fp32", + layout_a=la, layout_b=lb, layout_c=lc, + tile_m=128, tile_n=128, tile_k=32, + wave_m=2, wave_n=2, wave_k=1, + warp_tile_m=32, warp_tile_n=32, warp_tile_k=16, + pipeline="compv4", scheduler="intrawave", epilogue="cshuffle", + pad_m=True, pad_n=True, pad_k=True, persistent=False, + variant="multi_d", elementwise_op="MultiDAdd", + num_d_tensors=num_d, d_layout=ld, + gfx_arch=gfx_arch, + ) + so_paths = setup_multiple_gemm_dispatchers( + [cfg], verbose=log.isEnabledFor(logging.DEBUG) + ) + so = so_paths[0] + if so is None: + return FAIL, f"multi_d/fp16/{layout}: kernel failed to build" + + M, N, K = 512, 512, 512 + problem = MultiDGemmProblem(M=M, N=N, K=K, num_d=num_d) + + rng = np.random.default_rng(11) + A = rng.uniform(-1.0, 1.0, (M, K)).astype(np.float32) + B = rng.uniform(-1.0, 1.0, (K, N)).astype(np.float32) + Ds = [rng.uniform(-1.0, 1.0, (M, N)).astype(np.float32) + for _ in range(num_d)] + + runner = GpuMultiDGemmRunner(so) + if runner.num_d_tensors != num_d: + return FAIL, (f"multi_d/fp16/{layout}: kernel D-count mismatch " + f"({runner.num_d_tensors} != {num_d})") + result = runner.run(A.astype(np.float16), B.astype(np.float16), + [d.astype(np.float16) for d in Ds], problem) + if result.status != 0: + return FAIL, f"multi_d/fp16/{layout}: run status={result.status}" + if result.time_ms <= 0.0: + return FAIL, (f"multi_d/fp16/{layout}: time_ms={result.time_ms:.4f} " + f"is not positive") + + # Reference at fp16 input precision, accumulated + fused in fp32: + # E = A@B + D0 + D1 (MultiDAdd). + A_q = A.astype(np.float16).astype(np.float32) + B_q = B.astype(np.float16).astype(np.float32) + E_ref = A_q @ B_q + for d in Ds: + E_ref = E_ref + d.astype(np.float16).astype(np.float32) + + E_got = np.asarray(result.output).astype(np.float32) + if np.all(E_got == 0.0): + return FAIL, f"multi_d/fp16/{layout}: output all-zero" + if not np.all(np.isfinite(E_got)): + return FAIL, f"multi_d/fp16/{layout}: output NaN/Inf" + mre = _max_rel_err(E_got, E_ref) + if mre > _TOL: + return FAIL, (f"multi_d/fp16/{layout}: max_rel={mre:.4f} > tol={_TOL} " + f"(M={M} N={N} K={K})") + return PASS, (f"multi_d/fp16/{layout}: max_rel={mre:.4e}, " + f"time_ms={result.time_ms:.3f}") + + +# One entry per layout rather than one test that loops: each is its own hipcc +# build, and a per-layout PASS/FAIL row is what makes a single bad layout +# legible in the summary. +TESTS = [(layout, partial(_check_layout, layout)) for layout in _LAYOUTS] + + +def main(): + parser = argparse.ArgumentParser(description="Multi-D GEMM GPU correctness tests") + # No hardcoded default: it must stay possible to tell "user asked for + # gfx942" from "we are on a GPU-less box", so the skip below can fire. + parser.add_argument("--gfx", default=None, + help="GPU arch override (default: auto-detect)") + parser.add_argument("-v", "--verbose", action="store_true") + args = parser.parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(levelname)s: %(message)s", + ) + + gfx = args.gfx or _detect_arch() + if not gfx: + print("SKIP: no GPU detected (rocminfo); multi_d GPU tests skipped") + return SKIP_EXIT + if not _static_lib_present(): + print("SKIP: dispatcher static lib (libck_tile_dispatcher.a) not built; " + "multi_d is registry-routed and needs it") + return SKIP_EXIT + if shutil.which("hipcc") is None and not Path("/opt/rocm/bin/hipcc").exists(): + print("SKIP: hipcc not found; cannot build multi_d kernels") + return SKIP_EXIT + + results = [] + for name, fn in TESTS: + log.info("--- Running %s ---", name) + try: + status, detail = fn(gfx) + except Exception as exc: + status, detail = FAIL, f"{name}: exception: {exc}" + results.append((name, status, detail)) + log.info("[%s] %s", status, detail) + + print("\n=== Summary ===") + for name, status, detail in results: + print(f" [{status:4s}] {detail}") + passed = sum(1 for _, s, _ in results if s == PASS) + skipped = sum(1 for _, s, _ in results if s == SKIP) + failed = len(results) - passed - skipped + print(f"\n{passed}/{len(results) - skipped} passed" + + (f", {skipped} skipped" if skipped else "")) + + return 0 if failed == 0 else 1 if __name__ == "__main__": - unittest.main(verbosity=2) + raise SystemExit(main()) diff --git a/projects/composablekernel/dispatcher/tests/test_preshuffle_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_preshuffle_gpu_correctness.py index c763872ae370..c3376ea47cfd 100644 --- a/projects/composablekernel/dispatcher/tests/test_preshuffle_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_preshuffle_gpu_correctness.py @@ -9,7 +9,7 @@ Builds the preshuffle GEMM dispatcher .so (fp16 / rcr — the bridge's supported signature; permute_n is pinned False by BRIDGE_PERMUTE_N), runs a small GEMM on-device via GpuGemmRunner, and compares the GPU output to an fp32 numpy -reference within an fp16-appropriate tolerance. Skips cleanly (exit 0) when no +reference within an fp16-appropriate tolerance. Skips cleanly (exit 77) when no GPU / hipcc is available. The preshuffle kernel pre-permutes the B (weight) operand into a packed layout @@ -55,6 +55,10 @@ PASS = "PASS" FAIL = "FAIL" +# ctest SKIP_RETURN_CODE: main() returns this when the box cannot run the test +# at all, so the lane reports Skipped rather than a vacuous Passed or a Failed. +SKIP_EXIT = 77 + def _has_gpu() -> bool: try: @@ -185,13 +189,19 @@ def main() -> int: if not _has_gpu(): print("SKIP: no supported GPU detected (rocminfo); preshuffle GPU test skipped") - return 0 + return SKIP_EXIT gfx = args.gfx or _resolve_arch(None) log.info("Running preshuffle GEMM GPU correctness on %s", gfx) try: status, detail = _run_preshuffle_fp16(gfx) + except FileNotFoundError as exc: + # Same gate the pytest entry point applies: the on-device check needs the + # compiled dispatcher artifacts, and a unit-only stage runs without them. + # Absent artifacts is "cannot run here", not "the bridge is wrong". + print(f"SKIP: dispatcher not built; preshuffle GPU test skipped ({exc})") + return SKIP_EXIT except Exception as exc: # noqa: BLE001 status, detail = FAIL, f"preshuffle/fp16: exception: {exc}" diff --git a/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py index 4b36c6e34d16..f1c4c1d9eebe 100644 --- a/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_rowcolquant_gpu_correctness.py @@ -277,6 +277,12 @@ def test_rowcolquant_timing_positive(tmp_path): # Standalone runner (mirrors test_bquant_gpu_correctness.py style) # --------------------------------------------------------------------------- +# Mirrors the pytest cases above one-for-one. ctest runs this file as a script +# (not via `-m pytest`) so that a missing GPU exits 77 = Skipped, so anything +# absent from this list is not covered by ctest at all. +# +# test_*_timing_positive has no entry of its own because _run_one already fails +# the case when time_ms <= 0. TESTS = [ ("C4/fp8", lambda od, gfx: _run_one( "C4/fp8", default_fp8_config(gfx_arch=gfx), 128, 128, 192, @@ -284,6 +290,10 @@ def test_rowcolquant_timing_positive(tmp_path): ("C4/bf8", lambda od, gfx: _run_one( "C4/bf8", default_bf8_config(gfx_arch=gfx), 128, 128, 192, *_make_inputs(128, 128, 192, "bf8"), Path(od), gfx_arch=gfx)), + # Non-square M/N/K to stress stride math. + ("rect/fp8", lambda od, gfx: _run_one( + "rect/fp8", default_fp8_config(gfx_arch=gfx), 64, 256, 128, + *_make_inputs(64, 256, 128, "fp8"), Path(od), gfx_arch=gfx)), ] diff --git a/projects/composablekernel/dispatcher/tests/test_streamk_registry.py b/projects/composablekernel/dispatcher/tests/test_streamk_registry.py index 5e6c92b9d428..e93b173140f2 100644 --- a/projects/composablekernel/dispatcher/tests/test_streamk_registry.py +++ b/projects/composablekernel/dispatcher/tests/test_streamk_registry.py @@ -45,6 +45,15 @@ SKIP = 77 # ctest SKIP_RETURN_CODE +# Arches the Stream-K dispatcher is validated on. The gate matters because the +# arch can arrive from --arch (CMakeLists.txt passes the arch the build was +# configured with) and so bypass detect_arch() entirely. Without it, a box whose +# arch merely *has* a GPU -- gfx90a, say -- builds and runs the whole matrix and +# reports failures: fp16/bf16/fp8 pass there, but every bf8 case aborts with +# HSA_STATUS_ERROR_MEMORY_APERTURE_VIOLATION. That is an unsupported-arch result, +# not a regression, and it belongs in the Skipped column. +SUPPORTED_ARCHS = ("gfx942", "gfx950") + # One tile config, all three reduction strategies. TILE = "128x128x64_2x2x1_32x32x16" TILE_CONFIG_JSON = json.dumps( @@ -135,6 +144,12 @@ def main(): if not arch: print("SKIP: no GPU / could not detect gfx arch") return SKIP + # Strip the feature suffixes of a full target triple ("gfx950:sramecc+:xnack-") + # before matching, so a configured-target string is gated the same as a bare arch. + if arch.split(":", 1)[0] not in SUPPORTED_ARCHS: + print(f"SKIP: {arch} is not a supported Stream-K arch " + f"(supported: {', '.join(SUPPORTED_ARCHS)})") + return SKIP print(f"Stream-K registry test on {arch} @ {args.m}x{args.n}x{args.k}") inc = ["-I", str(CK_DIR / "include"), "-I", str(DISPATCHER_DIR / "include")] diff --git a/projects/composablekernel/dispatcher/tests/test_tensor_quant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_tensor_quant_gpu_correctness.py index 564328777a12..4fc3e920caa9 100644 --- a/projects/composablekernel/dispatcher/tests/test_tensor_quant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_tensor_quant_gpu_correctness.py @@ -28,11 +28,20 @@ Run: python3 -m pytest test_tensor_quant_gpu_correctness.py -v python3 test_tensor_quant_gpu_correctness.py # standalone + python3 test_tensor_quant_gpu_correctness.py --gfx gfx942 + +The standalone path is what ctest and the Jenkins lane drive; it exits 77 (see +SKIP_EXIT) when the box cannot run the test at all, so an unrunnable runner is +recorded as Skipped rather than as a vacuous Passed or a spurious Failed. """ +import argparse +import logging +import subprocess import sys import tempfile from pathlib import Path +from typing import Optional import numpy as np import pytest @@ -73,6 +82,37 @@ def _find_python_dir() -> Path: _detect_gpu_arch, ) +log = logging.getLogger(__name__) + +PASS = "PASS" +FAIL = "FAIL" +SKIP_EXIT = 77 # ctest SKIP_RETURN_CODE; see main() + +# TensorQuant is a block-scale fp8/bf8 op: it needs native fp8 MFMA. A GPU +# outside this set clears conftest's gpu_available() probe and then fails at +# hipcc, which reads as a correctness FAIL rather than "this box cannot run it". +# Matches SUPPORTED_ARCHS in the grouped tensorquant/rowcolquant GPU tests. +_SUPPORTED_ARCHS = ("gfx942", "gfx950") + + +def _has_hipcc() -> bool: + try: + subprocess.run(["hipcc", "--version"], capture_output=True, timeout=10) + return True + except Exception: + return False + + +def _safe_detect_arch() -> Optional[str]: + """_detect_gpu_arch() raises on absence/unsupported; the gates want None.""" + try: + return _detect_gpu_arch() + except Exception: + return None + + +_GFX_ARCH = _safe_detect_arch() + # Thin local aliases so the case body reads the same as before (the codecs are # the single canonical copies from conftest.py). @@ -99,8 +139,11 @@ def reference_tensor_quant_gemm(A_dec, B_dec, AQ: float, BQ: float) -> np.ndarra # ============================================================================= -def _run_case(dtype: str, M: int, N: int, K: int, out_dir: Path): - arch = _detect_gpu_arch() +def _run_case(dtype: str, M: int, N: int, K: int, out_dir: Path, + arch: Optional[str] = None): + # main() resolves the arch once (honouring --gfx) and passes it down; the + # pytest entry points leave it None and take the detected one. + arch = arch or _detect_gpu_arch() make_cfg = default_fp8_config if dtype == "fp8" else default_bf8_config config = make_cfg(arch) @@ -147,10 +190,19 @@ def _run(so_path): _SKIP_NO_GPU = pytest.mark.skipif(not _have_gpu(), reason="no ROCm GPU detected") _SKIP_NO_MLD = pytest.mark.skipif(not _have_ml_dtypes(), reason="ml_dtypes not installed") +# gpu_available() only says "a ROCm GPU is present". Without this second gate a +# gfx90a box runs the fp8 build and reports a red FAIL for hardware that was +# never in scope. +_SKIP_BAD_ARCH = pytest.mark.skipif( + _GFX_ARCH not in _SUPPORTED_ARCHS, + reason=f"TensorQuant needs native fp8 ({'/'.join(_SUPPORTED_ARCHS)}); " + f"detected {_GFX_ARCH}", +) @_SKIP_NO_GPU @_SKIP_NO_MLD +@_SKIP_BAD_ARCH @pytest.mark.parametrize("dtype", ["fp8", "bf8"]) def test_tensor_quant_gpu_matches_reference(dtype, tmp_path): max_rel, _ = _run_case(dtype, M=256, N=256, K=512, out_dir=tmp_path) @@ -159,22 +211,91 @@ def test_tensor_quant_gpu_matches_reference(dtype, tmp_path): @_SKIP_NO_GPU @_SKIP_NO_MLD +@_SKIP_BAD_ARCH def test_tensor_quant_gpu_not_all_zeros(tmp_path): _run_case("fp8", M=256, N=256, K=512, out_dir=tmp_path) -if __name__ == "__main__": - if not _have_gpu(): - print("SKIP: no GPU"); raise SystemExit(0) +# ============================================================================= +# Standalone entry point (the one ctest and Jenkins drive) +# ============================================================================= + + +def _case(dtype: str): + """Build a never-raising case fn returning (status, detail). + + The TESTS-loop convention is that a case reports its own verdict; main()'s + ``except Exception`` is a backstop for genuine bugs, not the normal path. + """ + def run(out_dir: Path, arch: str): + max_rel, time_ms = _run_case(dtype, 256, 256, 512, out_dir, arch) + if max_rel > 0.05: + return FAIL, f"tensor_quant {dtype}: max_rel={max_rel:.4f} > tol=0.05" + return PASS, (f"tensor_quant {dtype}: max_rel={max_rel:.4f} " + f"time_ms={time_ms:.3f}") + return run + + +TESTS = [(f"tensor_quant_{dt}", _case(dt)) for dt in ("fp8", "bf8")] + + +def main() -> int: + parser = argparse.ArgumentParser( + description="TensorQuant (non-grouped) GPU correctness tests") + # No hardcoded default: an explicit --gfx is an override the caller is + # trusted on, but falling back to a fixed arch would make a CPU-only or + # gfx90a box compile for gfx950 and crash instead of skipping. + parser.add_argument("--gfx", default=None, + help=f"GPU arch override (default: auto-detect; " + f"{'/'.join(_SUPPORTED_ARCHS)} only)") + parser.add_argument("-v", "--verbose", action="store_true") + parser.add_argument("--output-dir", type=Path, default=None) + args = parser.parse_args() + + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(levelname)s: %(message)s", + ) + + # The pytest path gates via the skipif marks above; this standalone path is + # what CI drives, so it needs the same gates spelled out. SKIP_EXIT keeps a + # clean skip distinguishable from a failure -- ctest maps it via + # SKIP_RETURN_CODE and the Jenkins lane via its run_ok helper. + gfx = args.gfx or _GFX_ARCH + if not gfx: + print("SKIP: no supported GPU detected (rocm_agent_enumerator)") + return SKIP_EXIT + if gfx not in _SUPPORTED_ARCHS: + print(f"SKIP: TensorQuant needs native fp8 " + f"({'/'.join(_SUPPORTED_ARCHS)}); detected {gfx}") + return SKIP_EXIT + if not _has_hipcc(): + print("SKIP: hipcc not found in PATH; cannot JIT-compile kernels") + return SKIP_EXIT if not _have_ml_dtypes(): - print("SKIP: ml_dtypes not installed"); raise SystemExit(0) - d = Path(tempfile.mkdtemp(prefix="tensor_quant_gputest_")) - ok = True - for dt in ("fp8", "bf8"): + print("SKIP: ml_dtypes not installed; fp8/bf8 encoding unavailable") + return SKIP_EXIT + + out_dir = args.output_dir or Path(tempfile.mkdtemp(prefix="tensor_quant_gputest_")) + log.info("Kernel output dir: %s", out_dir) + + results = [] + for name, fn in TESTS: + log.info("--- Running %s ---", name) try: - mr, t = _run_case(dt, 256, 256, 512, d) - print(f"PASS tensor_quant {dt}: max_rel={mr:.4f} time_ms={t:.3f}") - except Exception as e: - ok = False - print(f"FAIL tensor_quant {dt}: {e}") - raise SystemExit(0 if ok else 1) + status, detail = fn(out_dir, gfx) + except Exception as exc: # noqa: BLE001 + status, detail = FAIL, f"{name}: exception: {exc}" + results.append((status, detail)) + log.info("[%s] %s", status, detail) + + print("\n=== Summary ===") + passed = sum(1 for s, _ in results if s == PASS) + for status, detail in results: + print(f" [{status:4s}] {detail}") + print(f"\n{passed}/{len(results)} passed") + return 0 if passed == len(results) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py index 9d797acfc290..426da979811f 100644 --- a/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_tensorquant_gpu_correctness.py @@ -275,6 +275,12 @@ def test_tensorquant_timing_positive(tmp_path): # Standalone runner # --------------------------------------------------------------------------- +# Mirrors the pytest cases above one-for-one. ctest runs this file as a script +# (not via `-m pytest`) so that a missing GPU exits 77 = Skipped, so anything +# absent from this list is not covered by ctest at all. +# +# test_*_timing_positive has no entry of its own because _run_one already fails +# the case when time_ms <= 0. TESTS = [ ("C4/fp8", lambda od, gfx: _run_one( "C4/fp8", default_fp8_config(gfx_arch=gfx), 128, 128, 192, @@ -282,6 +288,10 @@ def test_tensorquant_timing_positive(tmp_path): ("C4/bf8", lambda od, gfx: _run_one( "C4/bf8", default_bf8_config(gfx_arch=gfx), 128, 128, 192, *_make_inputs(128, 128, 192, "bf8"), Path(od), gfx_arch=gfx)), + # Non-square M/N/K to stress stride math. + ("rect/fp8", lambda od, gfx: _run_one( + "rect/fp8", default_fp8_config(gfx_arch=gfx), 64, 256, 128, + *_make_inputs(64, 256, 128, "fp8"), Path(od), gfx_arch=gfx)), ] diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 913e45549fc1..d89b05fd3fd4 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1728,8 +1728,11 @@ def dispatcherSweepDtypesFor(String arch) { // // layouts are deliberately omitted: the runner picks per-variant defaults // (multi_d/multi_abd are fp16/rcrr-only, so naming the standard 4x4 matrix here -// would enumerate configs that variant cannot build). multi_d likewise takes the -// runner's default --elementwise-op PassThrough. +// would enumerate configs that variant cannot build). --elementwise-op is +// likewise omitted, so multi_d sweeps the MultiDAdd/MultiDMultiply x num_d +// {1,2} matrix from gemm_multi_d/configs/default_ci_config.json. Passing +// PassThrough here would make the lane vacuous: it discards the D tensors on +// both the device and reference sides, so the fusion under test is never run. // // dtypes are omitted for the same reason -- except for "grouped", whose // _VARIANT_DEFAULTS entry in test_gemm_search_space.py is the full @@ -1847,15 +1850,17 @@ def runDispatcherCorrectnessTests(String arch, String compiler) { run_ok python3 ../dispatcher/tests/test_batched_gemm_gpu_correctness.py --gfx ${arch} && \ run_ok python3 ../dispatcher/tests/test_batched_contraction_gpu_correctness.py --gfx ${arch} && \ python3 ../dispatcher/tests/test_grouped_gemm_gpu_correctness.py && \ - python3 ../dispatcher/tests/test_multi_d_gpu_correctness.py && \ + run_ok python3 ../dispatcher/tests/test_multi_d_gpu_correctness.py --gfx ${arch} && \ python3 ../dispatcher/tests/test_multi_abd_gpu_correctness.py && \ run_ok python3 ../dispatcher/tests/test_rowcolquant_gpu_correctness.py --gfx ${arch} && \ run_ok python3 ../dispatcher/tests/test_tensorquant_gpu_correctness.py --gfx ${arch}""" // The grouped/multi_d/multi_abd tests are the bridge-level companions to the // --variant sweeps below: the sweep exercises the search space, they exercise // the ctypes bridge. They were registered in ctest but never invoked from - // here. All three are unittest-based, take no --gfx, and exit 0 on an - // internal skip, so they need no run_ok. + // here. grouped_gemm and multi_abd are unittest-based, take no --gfx, and + // exit 0 on an internal skip, so they need no run_ok. multi_d is + // script-style: it takes --gfx and exits 77 on a clean skip, so it must be + // wrapped -- a bare 77 would break the && chain and fail the lane. // // rowcolquant and tensorquant sit here rather than in the gfx950 block // because, unlike the other quant ops, both support gfx942 as well -- @@ -1883,10 +1888,14 @@ def runDispatcherCorrectnessTests(String arch, String compiler) { run_ok python3 ../dispatcher/tests/test_streamk_registry.py --arch ${arch} --datatypes fp16 --layouts rcr && \ run_ok python3 ../dispatcher/tests/test_streamk_gpu_correctness.py --gfx ${arch}""" } - // bquant is still gfx950-only: its config builders hardcode warp_tile_k - // tuned for gfx950 (128), and on gfx942 get_k_warp_tile() takes the other - // branch of CK_GFX950_SUPPORT, so such a kernel builds and runs and returns - // zeros. Widening it means porting abquant's arch-aware _preshuffleb_warp_tile_k. + // bquant stays gfx950-only, but no longer for the original reason: its config + // builders used to hardcode the gfx950 warp_tile_k (128), which on gfx942 took + // the other branch of CK_GFX950_SUPPORT and produced a kernel that built, ran, + // and returned zeros. grouped_gemm_bquant_utils is arch-aware now + // (_fp8_warp_tile_k / _preshuffleb_warp_tile_k), so the gate is held only + // pending one validating run on gfx942 -- not by a known defect. Flipping it + // means widening SUPPORTED_ARCHS in test_bquant_gpu_correctness.py in the same + // change; the two must not drift apart. // This is why the lane fans out to a gfx950 node as well as gfx942 -- see the // stage comment in the Jenkinsfile. // diff --git a/projects/composablekernel/tile_engine/ops/gemm/gemm_multi_abd/configs/default_ci_config.json b/projects/composablekernel/tile_engine/ops/gemm/gemm_multi_abd/configs/default_ci_config.json index 99902697f822..b021261c6010 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/gemm_multi_abd/configs/default_ci_config.json +++ b/projects/composablekernel/tile_engine/ops/gemm/gemm_multi_abd/configs/default_ci_config.json @@ -88,5 +88,13 @@ ] } }, + "multi_abd_config": { + "num_a_tensors": 2, + "num_b_tensors": 2, + "num_d_tensors": 2, + "a_elementwise_op": "MultiDAdd", + "b_elementwise_op": "MultiDAdd", + "cde_elementwise_op": "MultiDAdd" + }, "k_block_per_cu": 1 } \ No newline at end of file From 7d3e2696650d60899562a8736bf83674e71b87ee Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Tue, 1 Sep 2026 21:29:57 -0400 Subject: [PATCH 37/51] docs(ck-tile): correct the bquant arch claim in the coverage summary The lane's coverage summary listed grouped_gemm_bquant C4 as running on both arches, contradicting both the gate 120 lines below it (if (arch == "gfx950")) and the rationale comment attached to that gate. The Jenkinsfile parameter description already said gfx950-only; this was the last of the three places still out of step. Comment only, no behaviour change. Co-Authored-By: Claude Opus 5 --- projects/composablekernel/groovy/vars/ck.groovy | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index d89b05fd3fd4..530bab432bcc 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1779,8 +1779,11 @@ def dispatcherVariantCmd(String arch, String variant) { // batched_gemm, batched_contraction (fp16+bf16+fp32), stream_k, // grouped_gemm_rowcolquant, grouped_gemm_tensorquant, // grouped_gemm_aquant, grouped_gemm_abquant (both arches), -// grouped_gemm_bquant C4 (fp8/bf8, both arches), -// grouped_gemm_bquant H3 (mx_*) + mx_gemm (gfx950 only, ISA limitation). +// grouped_gemm_bquant C4 (fp8/bf8) + C (fp8i4/bf8i4) + H3 (mx_*), +// mx_gemm. +// The last two lines are gfx950 only -- mx_gemm and bquant H3 for an ISA +// limitation (no scale-MFMA builtins on gfx942), bquant C4/C pending one +// validating gfx942 run. See the gate comment further down. // // Operators NOT covered by this lane (no dispatcher bridge on develop yet): // preshuffled_gemm, gemm_aquant, gemm_abquant, gemm_bquant, From ef5618a6609eb207531063a1dc558752e7b7e573 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Wed, 2 Sep 2026 14:35:06 -0400 Subject: [PATCH 38/51] Code Cleanup --- .../ctypes/streamk_gemm_ctypes_lib.cpp | 7 ++--- .../dispatcher/python/fmha_utils.py | 7 ----- .../composablekernel/groovy/vars/ck.groovy | 28 ++++++++++++------- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/projects/composablekernel/dispatcher/bindings/ctypes/streamk_gemm_ctypes_lib.cpp b/projects/composablekernel/dispatcher/bindings/ctypes/streamk_gemm_ctypes_lib.cpp index 37ed77230ef6..11b7fc44f005 100644 --- a/projects/composablekernel/dispatcher/bindings/ctypes/streamk_gemm_ctypes_lib.cpp +++ b/projects/composablekernel/dispatcher/bindings/ctypes/streamk_gemm_ctypes_lib.cpp @@ -14,11 +14,8 @@ * * static float launch(const ck_tile::StreamKHostArgs& args, const stream_config& stream); * - * which allocates the reduction workspace internally (DeviceMem). This lib is - * reduction-strategy agnostic: the strategy is a codegen-time template constant - * baked into SelectedKernel (unified_gemm_codegen.py), and GetWorkSpaceSize - * returns 0 for Atomic and a non-zero size for Linear/Tree, so the single - * allocate-and-launch path below serves all three. The single-problem registry path + * which allocates the reduction workspace internally (DeviceMem) and uses the + * Atomic reduction strategy. The single-problem registry path * (g_dispatcher->run / GemmHostArgs) and the generated_tile_backend wrapper both * hard-code the plain GemmHostArgs launch, so this lib bypasses the registry and * calls SelectedKernel::launch(args, stream) directly, reporting the kernel name diff --git a/projects/composablekernel/dispatcher/python/fmha_utils.py b/projects/composablekernel/dispatcher/python/fmha_utils.py index dcd67612f63c..5d3d0854960b 100644 --- a/projects/composablekernel/dispatcher/python/fmha_utils.py +++ b/projects/composablekernel/dispatcher/python/fmha_utils.py @@ -1236,13 +1236,6 @@ def fmha_compile_flags(arch: str, hipcc: str = "", family: str = "") -> List[str ] if arch.startswith("gfx9"): flags.append("-DCK_TILE_FMHA_FWD_FAST_EXP2=1") - # NOTE: OCP fp8 is forced on every gfx9 arch, including gfx942 whose CK - # default is FNUZ (see dispatcher_common.fp8_uses_ocp). That is not the - # host/device split-brain the GEMM bridges had -- the define lands on both - # compiler passes, so they agree -- but it does mean fmha fp8 on gfx942 - # runs a software OCP codec rather than the arch-native FNUZ one. - # Left as-is deliberately: flipping it is a numeric change to an operator - # outside this lane's coverage matrix and needs its own verification pass. flags.append("-DCK_TILE_USE_OCP_FP8") flags.append("-DCK_GFX950_SUPPORT") flags.append("-DCK_USE_GFX950") diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 530bab432bcc..21431056dc83 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1664,7 +1664,9 @@ def runDispatcherPerfTests(String compiler, String gpuTarget = "gfx942") { // reasoning as runDispatcherCorrectnessTests: execute_cmd runs from // projects/composablekernel/build while archiveArtifacts resolves against // the workspace root, so the build-relative path is spelled out in full. - archiveArtifacts artifacts: "projects/composablekernel/build/*_results.json", + // dispatcher_* prefix matches the correctness lane and avoids accidentally + // capturing tile_engine benchmark JSONs that share the same build dir. + archiveArtifacts artifacts: "projects/composablekernel/build/dispatcher_*_results.json", allowEmptyArchive: true } } @@ -1852,18 +1854,19 @@ def runDispatcherCorrectnessTests(String arch, String compiler) { run_ok python3 ../dispatcher/tests/test_gemm_parity.py && \ run_ok python3 ../dispatcher/tests/test_batched_gemm_gpu_correctness.py --gfx ${arch} && \ run_ok python3 ../dispatcher/tests/test_batched_contraction_gpu_correctness.py --gfx ${arch} && \ - python3 ../dispatcher/tests/test_grouped_gemm_gpu_correctness.py && \ + run_ok python3 ../dispatcher/tests/test_grouped_gemm_gpu_correctness.py && \ run_ok python3 ../dispatcher/tests/test_multi_d_gpu_correctness.py --gfx ${arch} && \ - python3 ../dispatcher/tests/test_multi_abd_gpu_correctness.py && \ + run_ok python3 ../dispatcher/tests/test_multi_abd_gpu_correctness.py && \ run_ok python3 ../dispatcher/tests/test_rowcolquant_gpu_correctness.py --gfx ${arch} && \ run_ok python3 ../dispatcher/tests/test_tensorquant_gpu_correctness.py --gfx ${arch}""" // The grouped/multi_d/multi_abd tests are the bridge-level companions to the // --variant sweeps below: the sweep exercises the search space, they exercise // the ctypes bridge. They were registered in ctest but never invoked from - // here. grouped_gemm and multi_abd are unittest-based, take no --gfx, and - // exit 0 on an internal skip, so they need no run_ok. multi_d is - // script-style: it takes --gfx and exits 77 on a clean skip, so it must be - // wrapped -- a bare 77 would break the && chain and fail the lane. + // here. All three are wrapped with run_ok: multi_d is script-style and exits + // 77 on a clean skip (a bare 77 would break the && chain), and grouped_gemm / + // multi_abd are unittest-based but still need run_ok to guard against exit 1 + // from environment failures (bad import, setUpClass error) that would kill the + // rest of the chain without running the quant operators. // // rowcolquant and tensorquant sit here rather than in the gfx950 block // because, unlike the other quant ops, both support gfx942 as well -- @@ -1908,12 +1911,12 @@ def runDispatcherCorrectnessTests(String arch, String compiler) { // expand_sweep -- mx_gemm_utils exposes only default_fp8_config() / // default_fp4_config() -- so it cannot join the --variant sweep above and // instead runs the two-config smoke test merged in #10132. That test is - // unittest-based and self-gates to gfx950, so it takes no --gfx and needs no - // run_ok (unlike its neighbours, which do). + // unittest-based and self-gates to gfx950, so it takes no --gfx. It is still + // wrapped with run_ok -- see the NOTE below for why every test in this chain is. if (arch == "gfx950") { execute_cmd += """ && \ run_ok python3 ../dispatcher/tests/test_bquant_gpu_correctness.py --gfx ${arch} && \ - python3 ../dispatcher/tests/test_mx_gemm_gpu_correctness.py""" + run_ok python3 ../dispatcher/tests/test_mx_gemm_gpu_correctness.py""" } // aquant runs on both archs. Its non-preshuffleaq builders use standard fp8 // MFMA (warp_tile_k=32), present on gfx942. _preshuffleaq_warp_tile_k() is @@ -1930,6 +1933,11 @@ def runDispatcherCorrectnessTests(String arch, String compiler) { // pending one green run, and reports that as a per-case SKIP, not a 77. execute_cmd += """ && \ run_ok python3 ../dispatcher/tests/test_abquant_gpu_correctness.py --gfx ${arch}""" + // NOTE: run_ok is applied to every script-style test AND every unittest-based + // test above. Unittest-based tests exit 0 on an internal skipTest, but a bad + // import or setUpClass crash still exits 1, which would break the && chain and + // silently drop every subsequent operator. run_ok costs nothing and makes the + // chain resilient to environment failures regardless of test style. try { buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) } finally { From 53f78864429c5b06adbb2b75bc82f515033eb3f9 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Wed, 2 Sep 2026 15:10:44 -0400 Subject: [PATCH 39/51] Enabling manual trigger for CI Phases --- projects/composablekernel/Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index 90b98e9afa48..48f466703285 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -191,11 +191,11 @@ pipeline { description: "Run the tile_engine_gemm tests (default: OFF)") booleanParam( name: "RUN_DISPATCHER_PERF_TESTS", - defaultValue: false, + defaultValue: true, description: "Smoke-tier benchmark of ahead-of-time tile_engine GEMM instances on gfx942 + gfx950, one problem size and 64 instances per operator: gemm_universal, batched_gemm, batched_contraction, gemm_streamk, grouped_gemm at fp16/rcr and gemm_multi_d, gemm_multi_abd at fp16/rcrr (4-char layout) and grouped_gemm_rowcolquant, grouped_gemm_tensorquant at fp8/rcr on gfx942; gemm_aquant, gemm_bquant, gemm_abquant, mx_gemm at fp8/rcr and gemm_streamk at fp16/rcr on gfx950. mx_gemm is gfx950-only (its CMakeLists filters targets on a ^gfx950 regex and it has no fp16 path); gemm_streamk runs on both to match the correctness lane and to confirm its recently-widened DESIRED_TARGETS on gfx950. Covers the AOT instance path for every operator with a dispatcher bridge; RUN_DISPATCHER_CORRECTNESS_TESTS covers the JIT path for the same set (default: OFF)") booleanParam( name: "RUN_DISPATCHER_CORRECTNESS_TESTS", - defaultValue: false, + defaultValue: true, description: "Smoke-tier verification of JIT-compiled dispatcher kernels on gfx942 + gfx950: gemm_universal sweep at budget 64 plus parity, grouped_gemm, multi_d (all 4 layouts), multi_abd, batched_gemm, batched_contraction (fp16+bf16+fp32), stream_k, grouped_gemm_rowcolquant, grouped_gemm_tensorquant, grouped_gemm_aquant, grouped_gemm_abquant on both arches; mx_gemm + grouped_gemm_bquant (C4 fp8/bf8, C fp8i4/bf8i4, H3 mx_*) on gfx950 only — H3 for an ISA limitation, the rest pending a validating gfx942 run; rcr layouts only for quant operators. Does NOT cover preshuffled_gemm, gemm_aquant, gemm_abquant, gemm_bquant, gemm_rowcolquant, gemm_tensorquant (block_scale_gemm), or batched_contraction_multi_abd — those operators have no dispatcher bridge on develop yet (default: OFF)") booleanParam( name: "BUILD_INSTANCES_ONLY", From b0b7cc4592d6c4e07b988943aa5c6e88c6302c4d Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Wed, 2 Sep 2026 15:28:51 -0400 Subject: [PATCH 40/51] Changes to Jenkins and Groovy --- projects/composablekernel/Jenkinsfile | 4 +- .../composablekernel/groovy/vars/ck.groovy | 242 +++++++++--------- 2 files changed, 129 insertions(+), 117 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index 48f466703285..d571ecd5b61c 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -192,11 +192,11 @@ pipeline { booleanParam( name: "RUN_DISPATCHER_PERF_TESTS", defaultValue: true, - description: "Smoke-tier benchmark of ahead-of-time tile_engine GEMM instances on gfx942 + gfx950, one problem size and 64 instances per operator: gemm_universal, batched_gemm, batched_contraction, gemm_streamk, grouped_gemm at fp16/rcr and gemm_multi_d, gemm_multi_abd at fp16/rcrr (4-char layout) and grouped_gemm_rowcolquant, grouped_gemm_tensorquant at fp8/rcr on gfx942; gemm_aquant, gemm_bquant, gemm_abquant, mx_gemm at fp8/rcr and gemm_streamk at fp16/rcr on gfx950. mx_gemm is gfx950-only (its CMakeLists filters targets on a ^gfx950 regex and it has no fp16 path); gemm_streamk runs on both to match the correctness lane and to confirm its recently-widened DESIRED_TARGETS on gfx950. Covers the AOT instance path for every operator with a dispatcher bridge; RUN_DISPATCHER_CORRECTNESS_TESTS covers the JIT path for the same set (default: OFF)") + description: "Smoke-tier AOT benchmark on gfx942+gfx950: 64 instances/op, one problem size. gfx942: gemm_universal, batched_gemm, batched_contraction, gemm_streamk, grouped_gemm (fp16/rcr), gemm_multi_d, gemm_multi_abd (fp16/rcrr), grouped_gemm_rowcolquant, grouped_gemm_tensorquant (fp8/rcr). gfx950: gemm_aquant, gemm_bquant, gemm_abquant, mx_gemm (fp8/rcr), gemm_streamk (fp16/rcr). (default: OFF)") booleanParam( name: "RUN_DISPATCHER_CORRECTNESS_TESTS", defaultValue: true, - description: "Smoke-tier verification of JIT-compiled dispatcher kernels on gfx942 + gfx950: gemm_universal sweep at budget 64 plus parity, grouped_gemm, multi_d (all 4 layouts), multi_abd, batched_gemm, batched_contraction (fp16+bf16+fp32), stream_k, grouped_gemm_rowcolquant, grouped_gemm_tensorquant, grouped_gemm_aquant, grouped_gemm_abquant on both arches; mx_gemm + grouped_gemm_bquant (C4 fp8/bf8, C fp8i4/bf8i4, H3 mx_*) on gfx950 only — H3 for an ISA limitation, the rest pending a validating gfx942 run; rcr layouts only for quant operators. Does NOT cover preshuffled_gemm, gemm_aquant, gemm_abquant, gemm_bquant, gemm_rowcolquant, gemm_tensorquant (block_scale_gemm), or batched_contraction_multi_abd — those operators have no dispatcher bridge on develop yet (default: OFF)") + description: "Smoke-tier JIT correctness on gfx942+gfx950: gemm_universal (budget 64), grouped_gemm, multi_d, multi_abd, batched_gemm, batched_contraction, stream_k, grouped_gemm_rowcolquant/tensorquant/aquant/abquant on both arches; mx_gemm + grouped_gemm_bquant on gfx950 only (ISA limitation). Does NOT cover preshuffled_gemm, block_scale_gemm ops, or batched_contraction_multi_abd (no bridge on develop). (default: OFF)") booleanParam( name: "BUILD_INSTANCES_ONLY", defaultValue: false, diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 21431056dc83..e39f44a95089 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1404,11 +1404,9 @@ def runComprehensiveConvDatasetTests() { ) } -def runTileEngineBasicTests(String compiler) { - buildAndTest( - setup_args: "NO_CK_BUILD", - build_type: 'Release', - execute_cmd: """ +@NonCPS +String _tileEngineBasicCmd(String compiler) { + return """ cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ -D BUILD_CK_TILE_ENGINE="ON" \ -D CMAKE_CXX_COMPILER="${compiler}" \ @@ -1423,17 +1421,23 @@ def runTileEngineBasicTests(String compiler) { -D GEMM_PRESHUFFLE_DATATYPE="fp16;fp8;bf16;bf8" \ -D GEMM_PRESHUFFLE_LAYOUT="rcr" \ -D GEMM_PRESHUFFLE_CONFIG_FILE="default_ci_config.json" .. && \ - ninja -j${nthreads()} benchmark_gemm_universal_all benchmark_gemm_preshuffle_all benchmark_gemm_multi_d_all && \ + ninja -j\$(nproc) benchmark_gemm_universal_all benchmark_gemm_preshuffle_all benchmark_gemm_multi_d_all && \ python3 ../tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_universal_results.json && \ python3 ../tile_engine/ops/gemm/gemm_preshuffle/gemm_preshuffle_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_preshuffle_results.json && \ python3 ../tile_engine/ops/gemm/gemm_multi_d/gemm_multi_d_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_multi_d_results.json""" +} + +def runTileEngineBasicTests(String compiler) { + buildAndTest( + setup_args: "NO_CK_BUILD", + build_type: 'Release', + execute_cmd: _tileEngineBasicCmd(compiler) ) } -def runTileEngineGemmTests(String arch, String compiler) { - def execute_cmd - if (arch == "gfx942") { - execute_cmd = """ +@NonCPS +String _tileEngineGemmCmd_gfx942(String compiler) { + return """ cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ -D BUILD_CK_TILE_ENGINE="ON" \ -D CMAKE_CXX_COMPILER="${compiler}" \ @@ -1464,7 +1468,7 @@ def runTileEngineGemmTests(String arch, String compiler) { -D BATCHED_GEMM_DATATYPE="fp16" \ -D BATCHED_GEMM_LAYOUT="rcr" \ -D TILE_ENGINE_SAMPLING_TIER=daily .. && \ - ninja -j${nthreads()} benchmark_gemm_universal_all benchmark_gemm_preshuffle_all benchmark_gemm_multi_d_all benchmark_gemm_streamk_all benchmark_grouped_gemm_all benchmark_gemm_multi_abd_all benchmark_batched_contraction_all benchmark_gemm_rowcolquant_all benchmark_gemm_tensor_quant_all benchmark_grouped_gemm_rowcolquant_all benchmark_grouped_gemm_tensorquant_all benchmark_batched_gemm_all && \ + ninja -j\$(nproc) benchmark_gemm_universal_all benchmark_gemm_preshuffle_all benchmark_gemm_multi_d_all benchmark_gemm_streamk_all benchmark_grouped_gemm_all benchmark_gemm_multi_abd_all benchmark_batched_contraction_all benchmark_gemm_rowcolquant_all benchmark_gemm_tensor_quant_all benchmark_grouped_gemm_rowcolquant_all benchmark_grouped_gemm_tensorquant_all benchmark_batched_gemm_all && \ python3 ../tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_universal_results.json && \ python3 ../tile_engine/ops/gemm/gemm_preshuffle/gemm_preshuffle_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_preshuffle_results.json && \ python3 ../tile_engine/ops/gemm/gemm_multi_d/gemm_multi_d_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_multi_d_results.json && \ @@ -1476,8 +1480,11 @@ def runTileEngineGemmTests(String arch, String compiler) { python3 ../tile_engine/ops/gemm/grouped_gemm_quant/grouped_gemm_rowcolquant/grouped_gemm_rowcolquant_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json grouped_gemm_rowcolquant_results.json && \ python3 ../tile_engine/ops/gemm/grouped_gemm_quant/grouped_gemm_tensorquant/grouped_gemm_tensorquant_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json grouped_gemm_tensorquant_results.json && \ python3 ../tile_engine/ops/gemm/batched_gemm/batched_gemm_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json batched_gemm_results.json """ - } else if (arch == "gfx950") { - execute_cmd = """ +} + +@NonCPS +String _tileEngineGemmCmd_gfx950(String compiler) { + return """ cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ -D BUILD_CK_TILE_ENGINE="ON" \ -D CMAKE_CXX_COMPILER="${compiler}" \ @@ -1492,13 +1499,16 @@ def runTileEngineGemmTests(String arch, String compiler) { -D MX_GEMM_DATATYPE="fp4;fp8" \ -D MX_GEMM_LAYOUT="rcr" \ -D TILE_ENGINE_SAMPLING_TIER=daily .. && \ - ninja -j${nthreads()} benchmark_gemm_universal_all benchmark_gemm_preshuffle_all benchmark_gemm_multi_d_all && \ + ninja -j\$(nproc) benchmark_gemm_universal_all benchmark_gemm_preshuffle_all benchmark_gemm_multi_d_all && \ python3 ../tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_universal_results.json && \ python3 ../tile_engine/ops/gemm/gemm_preshuffle/gemm_preshuffle_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_preshuffle_results.json && \ python3 ../tile_engine/ops/gemm/gemm_multi_d/gemm_multi_d_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_multi_d_results.json && \ python3 ../tile_engine/ops/gemm/mx_gemm/mx_gemm_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json mx_gemm_results.json """ - } else if (arch == "gfx1201") { - execute_cmd = """ +} + +@NonCPS +String _tileEngineGemmCmd_gfx1201(String compiler) { + return """ cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ -D BUILD_CK_TILE_ENGINE="ON" \ -D CMAKE_CXX_COMPILER="${compiler}" \ @@ -1507,82 +1517,29 @@ def runTileEngineGemmTests(String arch, String compiler) { -D GEMM_UNIVERSAL_DATATYPE="fp16" \ -D GEMM_UNIVERSAL_LAYOUT="rcr;rrr;crr;ccr" \ -D TILE_ENGINE_SAMPLING_TIER=daily .. && \ - ninja -j${nthreads()} benchmark_gemm_universal_all && \ + ninja -j\$(nproc) benchmark_gemm_universal_all && \ python3 ../tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_universal_results.json""" +} + +def runTileEngineGemmTests(String arch, String compiler) { + def execute_cmd + if (arch == "gfx942") { + execute_cmd = _tileEngineGemmCmd_gfx942(compiler) + } else if (arch == "gfx950") { + execute_cmd = _tileEngineGemmCmd_gfx950(compiler) + } else if (arch == "gfx1201") { + execute_cmd = _tileEngineGemmCmd_gfx1201(compiler) } buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) } -// Benchmark ahead-of-time tile_engine instances at smoke coverage: one dtype, -// one layout, one problem size, and a fixed per-op AOT instance cap. Sized to -// catch codegen/build/launch breakage on demand without booking a GPU node for -// the length of a full sweep. -// -// Covers every operator that has a dispatcher bridge, so the AOT instance path -// gets the same operator coverage runDispatcherCorrectnessTests gives the JIT -// path. The two are gated separately and neither substitutes for the other: -// correctness JIT-builds each config with hipcc and checks it against a host -// reference, while this lane exercises tile_engine's codegen, instance builder -// and benchmark harness. -def runDispatcherPerfTests(String compiler, String gpuTarget = "gfx942") { - def samplingTier = "500" - def gemmDatatype = "fp16" - def gemmLayout = "rcr" - def problemSizes = '"1024,1024,1024"' - def problemConfigs = '"g=2;m=1024;n=1024;k=1024"' - // gemm_multi_d and gemm_multi_abd take 4-character layouts (A, B, D, E), - // not the 3-character rcr the other ops use. Passing rcr here is not a - // narrower selection, it is a parse error in the instance builder. - def multiLayout = "rcrr" - // Pin instances per op rather than letting TILE_ENGINE_SAMPLING_TIER divide - // itself up. That tier is a *total* budget split evenly across whichever ops - // have a non-empty _DATATYPE (tile_engine/ops/gemm/CMakeLists.txt:16-23), - // so every operator added below would silently shrink the coverage of the - // ones already here -- gfx942 alone would have gone 125 -> 55 instances per - // op. An explicit _MAX_INSTANCES wins over the allocated value (same - // file, :50-59), which keeps lane cost fixed and independent of how many ops - // the lane happens to build. The tier is still passed: each op also forwards - // it to the sampler as --tier, where it picks the sampling strategy. - // - // One dtype and one layout per op is also load-bearing, not just smoke - // sizing: the per-op budget is divided again across (dtype x layout) combos - // with integer truncation, so a multi-combo op can floor a combo to zero - // instances and build nothing. - def perOpInstances = "64" - // gemm_aquant and gemm_bquant run on gfx950 because their tests and utils - // modules self-gate to it -- SUPPORTED_ARCHS = ("gfx950",) and - // _DEFAULT_GFX_ARCH = "gfx950" -- not because gfx942 lacks fp8. gfx942 has - // fp8 (FNUZ) and is in DESIRED_TARGETS; widening these ops to it means - // relaxing those guards first. - // - // mx_gemm is on gfx950 for a harder reason: its CMakeLists filters targets - // with a literal regex, target MATCHES "^gfx950", so on any other arch the - // benchmark_mx_gemm_all target is never created and ninja fails on an - // unknown target rather than skipping. It also has no fp16 path at all - // (MX_GEMM_DATATYPE defaults to fp4;fp8), hence quantDatatype below. - // - // gemm_streamk is built on both arches. It used to be gfx942-only here - // because its DESIRED_TARGETS was "gfx90a;gfx942" with a TODO for gfx950; - // that list now includes gfx950, but the commit that widened it flagged the - // CMake change as the one part it could not exercise locally and left it for - // CI to confirm. Building it on gfx950 here is that confirmation, and it also - // matches runDispatcherCorrectnessTests, which already runs stream_k on both. - // If the gfx950 AOT build turns out to be broken, the DESIRED_TARGETS line is - // revertible on its own without touching this lane. - // - // Everything else -- gemm_universal, batched_gemm, batched_contraction, - // grouped_gemm, gemm_multi_d, gemm_multi_abd and the two grouped quant ops -- - // is CMake-legal on both, and sits on gfx942 to keep the two stages balanced. - def quantDatatype = "fp8" - def quantLayout = "rcr" - // Stream-K below is tile_engine/ops/gemm_streamk/ (top level), not - // tile_engine/ops/gemm/gemm_streamk/. Two paths exist and only the first is - // real: it holds the CMakeLists, the instance builder and the benchmark. The - // nested one contains a lone configs/default_config.json that nothing in the - // tree reads. Edit the top-level tree. - def execute_cmd +@NonCPS +String _dispatcherPerfCmd(String compiler, String gpuTarget, String samplingTier, + String gemmDatatype, String gemmLayout, String multiLayout, + String quantDatatype, String quantLayout, + String problemSizes, String problemConfigs, String perOpInstances) { if (gpuTarget == "gfx950") { - execute_cmd = """ + return """ cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ -D BUILD_CK_TILE_ENGINE="ON" \ -D CMAKE_CXX_COMPILER="${compiler}" \ @@ -1604,14 +1561,14 @@ def runDispatcherPerfTests(String compiler, String gpuTarget = "gfx942") { -D GEMM_STREAMK_LAYOUT="${gemmLayout}" \ -D GEMM_STREAMK_MAX_INSTANCES=${perOpInstances} \ -D TILE_ENGINE_SAMPLING_TIER=${samplingTier} .. && \ - ninja -j${nthreads()} benchmark_gemm_aquant_all benchmark_gemm_bquant_all benchmark_gemm_abquant_all benchmark_mx_gemm_all benchmark_gemm_streamk_all && \ + ninja -j\$(nproc) benchmark_gemm_aquant_all benchmark_gemm_bquant_all benchmark_gemm_abquant_all benchmark_mx_gemm_all benchmark_gemm_streamk_all && \ python3 ../tile_engine/ops/gemm/block_scale_gemm/gemm_aquant/gemm_aquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_aquant_results.json && \ python3 ../tile_engine/ops/gemm/block_scale_gemm/gemm_bquant/gemm_bquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_bquant_results.json && \ python3 ../tile_engine/ops/gemm/block_scale_gemm/gemm_abquant/gemm_abquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_abquant_results.json && \ python3 ../tile_engine/ops/gemm/mx_gemm/mx_gemm_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json mx_gemm_results.json && \ python3 ../tile_engine/ops/gemm_streamk/gemm_streamk_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_streamk_results.json""" - } else { - execute_cmd = """ + } + return """ cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ -D BUILD_CK_TILE_ENGINE="ON" \ -D CMAKE_CXX_COMPILER="${compiler}" \ @@ -1645,7 +1602,7 @@ def runDispatcherPerfTests(String compiler, String gpuTarget = "gfx942") { -D GROUPED_GEMM_TENSORQUANT_LAYOUT="${quantLayout}" \ -D GROUPED_GEMM_TENSORQUANT_MAX_INSTANCES=${perOpInstances} \ -D TILE_ENGINE_SAMPLING_TIER=${samplingTier} .. && \ - ninja -j${nthreads()} benchmark_gemm_universal_all benchmark_batched_gemm_all benchmark_batched_contraction_all benchmark_gemm_streamk_all benchmark_grouped_gemm_all benchmark_gemm_multi_d_all benchmark_gemm_multi_abd_all benchmark_grouped_gemm_rowcolquant_all benchmark_grouped_gemm_tensorquant_all && \ + ninja -j\$(nproc) benchmark_gemm_universal_all benchmark_batched_gemm_all benchmark_batched_contraction_all benchmark_gemm_streamk_all benchmark_grouped_gemm_all benchmark_gemm_multi_d_all benchmark_gemm_multi_abd_all benchmark_grouped_gemm_rowcolquant_all benchmark_grouped_gemm_tensorquant_all && \ python3 ../tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_universal_results.json && \ python3 ../tile_engine/ops/gemm/batched_gemm/batched_gemm_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json batched_gemm_results.json && \ python3 ../tile_engine/ops/gemm/batched_contraction/batched_contraction_benchmark.py . --problem-configs ${problemConfigs} --warmup 5 --repeat 5 --verbose --json batched_contraction_results.json && \ @@ -1655,7 +1612,76 @@ def runDispatcherPerfTests(String compiler, String gpuTarget = "gfx942") { python3 ../tile_engine/ops/gemm/gemm_multi_abd/gemm_multi_abd_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_multi_abd_results.json && \ python3 ../tile_engine/ops/gemm/grouped_gemm_quant/grouped_gemm_rowcolquant/grouped_gemm_rowcolquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json grouped_gemm_rowcolquant_results.json && \ python3 ../tile_engine/ops/gemm/grouped_gemm_quant/grouped_gemm_tensorquant/grouped_gemm_tensorquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json grouped_gemm_tensorquant_results.json""" - } +} + +// Benchmark ahead-of-time tile_engine instances at smoke coverage: one dtype, +// one layout, one problem size, and a fixed per-op AOT instance cap. Sized to +// catch codegen/build/launch breakage on demand without booking a GPU node for +// the length of a full sweep. +// +// Covers every operator that has a dispatcher bridge, so the AOT instance path +// gets the same operator coverage runDispatcherCorrectnessTests gives the JIT +// path. The two are gated separately and neither substitutes for the other: +// correctness JIT-builds each config with hipcc and checks it against a host +// reference, while this lane exercises tile_engine's codegen, instance builder +// and benchmark harness. +def runDispatcherPerfTests(String compiler, String gpuTarget = "gfx942") { + def samplingTier = "500" + def gemmDatatype = "fp16" + def gemmLayout = "rcr" + def problemSizes = '"1024,1024,1024"' + def problemConfigs = '"g=2;m=1024;n=1024;k=1024"' + // gemm_multi_d and gemm_multi_abd take 4-character layouts (A, B, D, E), + // not the 3-character rcr the other ops use. Passing rcr here is not a + // narrower selection, it is a parse error in the instance builder. + def multiLayout = "rcrr" + // Pin instances per op rather than letting TILE_ENGINE_SAMPLING_TIER divide + // itself up. That tier is a *total* budget split evenly across whichever ops + // have a non-empty _DATATYPE (tile_engine/ops/gemm/CMakeLists.txt:16-23), + // so every operator added below would silently shrink the coverage of the + // ones already here -- gfx942 alone would have gone 125 -> 55 instances per + // op. An explicit _MAX_INSTANCES wins over the allocated value (same + // file, :50-59), which keeps lane cost fixed and independent of how many ops + // the lane happens to build. The tier is still passed: each op also forwards + // it to the sampler as --tier, where it picks the sampling strategy. + // + // One dtype and one layout per op is also load-bearing, not just smoke + // sizing: the per-op budget is divided again across (dtype x layout) combos + // with integer truncation, so a multi-combo op can floor a combo to zero + // instances and build nothing. + def perOpInstances = "64" + // gemm_aquant and gemm_bquant run on gfx950 because their tests and utils + // modules self-gate to it -- SUPPORTED_ARCHS = ("gfx950",) and + // _DEFAULT_GFX_ARCH = "gfx950" -- not because gfx942 lacks fp8. gfx942 has + // fp8 (FNUZ) and is in DESIRED_TARGETS; widening these ops to it means + // relaxing those guards first. + // + // mx_gemm is on gfx950 for a harder reason: its CMakeLists filters targets + // with a literal regex, target MATCHES "^gfx950", so on any other arch the + // benchmark_mx_gemm_all target is never created and ninja fails on an + // unknown target rather than skipping. It also has no fp16 path at all + // (MX_GEMM_DATATYPE defaults to fp4;fp8), hence quantDatatype below. + // + // gemm_streamk is built on both arches. It used to be gfx942-only here + // because its DESIRED_TARGETS was "gfx90a;gfx942" with a TODO for gfx950; + // that list now includes gfx950, but the commit that widened it flagged the + // CMake change as the one part it could not exercise locally and left it for + // CI to confirm. Building it on gfx950 here is that confirmation, and it also + // matches runDispatcherCorrectnessTests, which already runs stream_k on both. + // If the gfx950 AOT build turns out to be broken, the DESIRED_TARGETS line is + // revertible on its own without touching this lane. + // + // Everything else -- gemm_universal, batched_gemm, batched_contraction, + // grouped_gemm, gemm_multi_d, gemm_multi_abd and the two grouped quant ops -- + // is CMake-legal on both, and sits on gfx942 to keep the two stages balanced. + def quantDatatype = "fp8" + def quantLayout = "rcr" + // Stream-K below is tile_engine/ops/gemm_streamk/ (top level), not + // tile_engine/ops/gemm/gemm_streamk/. Two paths exist and only the first is + // real: it holds the CMakeLists, the instance builder and the benchmark. The + // nested one contains a lone configs/default_config.json that nothing in the + // tree reads. Edit the top-level tree. + def execute_cmd = _dispatcherPerfCmd(compiler, gpuTarget, samplingTier, gemmDatatype, gemmLayout, multiLayout, quantDatatype, quantLayout, problemSizes, problemConfigs, perOpInstances) try { buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) } finally { @@ -1811,28 +1837,9 @@ def dispatcherVariantCmd(String arch, String variant) { // derivation from the compile-time layout // types (see plan Step 7). // --------------------------------------------------------------------------- -def runDispatcherCorrectnessTests(String arch, String compiler) { - def budget = 64 - // run_ok wraps the script-style tests, which signal a clean skip with exit - // 77. execute_cmd is one &&-joined string, so without this an unsupported - // arch or a missing ml_dtypes would abort the lane at that point and every - // later operator would silently never run. Only 77 is swallowed; any other - // non-zero exit still fails the lane. A function definition itself exits 0, - // so it chains cleanly ahead of cmake. - // - // "$@" || rc=$?, not "$@"; rc=$? -- and the difference is not cosmetic. - // buildAndTest emits `sh cmd` with no shebang, so the Durable Task plugin - // runs this under /bin/sh -xe and set -e is live. set -e is suppressed - // inside a function body only while the call is a NON-FINAL element of an - // && list; when run_ok is the last element, set -e applies inside the body - // and a bare "$@" exiting 77 kills the shell before rc=$? can run. Both - // arch chains below end on a run_ok (test_abquant_gpu_correctness on both), - // so a legitimate skip there would - // fail the lane. `cmd || rc=$?` puts the call in a tested context, which is - // immune to set -e regardless of where run_ok is called from. Keep it that - // way. rc=0 is a plain assignment, not `local`: /bin/sh is not bash and - // local is not POSIX. - def execute_cmd = """ +@NonCPS +String _dispatcherCorrectnessBaseCmd(String arch, String compiler, String dtypes, int budget) { + return """ run_ok() { rc=0; "\$@" || rc=\$?; if [ \$rc -eq 77 ]; then echo "SKIP(77): \$*"; return 0; fi; return \$rc; } && \ cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ -D CMAKE_CXX_COMPILER="${compiler}" \ @@ -1841,10 +1848,10 @@ def runDispatcherCorrectnessTests(String arch, String compiler) { -D CK_TILE_DISPATCHER=ON \ -D BUILD_DISPATCHER_BINDINGS=ON \ -D DISPATCHER_RULE_SET=tests .. && \ - ninja -j${nthreads()} ck_tile_dispatcher dispatcher_gemm_lib && \ + ninja -j\$(nproc) ck_tile_dispatcher dispatcher_gemm_lib && \ python3 ../dispatcher/tests/test_gemm_search_space.py \ --arch ${arch} \ - --dtypes ${dispatcherSweepDtypesFor(arch)} \ + --dtypes ${dtypes} \ --layouts rcr,rrr,crr,ccr \ --budget ${budget} \ --warmup 5 \ @@ -1859,6 +1866,11 @@ def runDispatcherCorrectnessTests(String arch, String compiler) { run_ok python3 ../dispatcher/tests/test_multi_abd_gpu_correctness.py && \ run_ok python3 ../dispatcher/tests/test_rowcolquant_gpu_correctness.py --gfx ${arch} && \ run_ok python3 ../dispatcher/tests/test_tensorquant_gpu_correctness.py --gfx ${arch}""" +} + +def runDispatcherCorrectnessTests(String arch, String compiler) { + def budget = 64 + def execute_cmd = _dispatcherCorrectnessBaseCmd(arch, compiler, dispatcherSweepDtypesFor(arch), budget) // The grouped/multi_d/multi_abd tests are the bridge-level companions to the // --variant sweeps below: the sweep exercises the search space, they exercise // the ctypes bridge. They were registered in ctest but never invoked from From 0ece186176328d5cc2237a34fce151d4de620fbc Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Wed, 2 Sep 2026 15:49:19 -0400 Subject: [PATCH 41/51] Changes to Jenkins file --- projects/composablekernel/Jenkinsfile | 88 ++++++++++++--------------- 1 file changed, 40 insertions(+), 48 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index d571ecd5b61c..e70bb762dc48 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -91,7 +91,7 @@ def showCompilerInfo(boolean deferBinaryInfo = false, String dockerImage = "") { //launch develop branch daily jobs CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;RUN_CK_TILE_FMHA_TESTS=true;RUN_PERFORMANCE_TESTS=true;FORCE_CI=true - 0 22 * * * % RUN_FULL_QA=true;DISABLE_DL_KERNELS=true;RUN_TILE_ENGINE_BASIC_TESTS=true;RUN_TILE_ENGINE_GEMM_TESTS=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true + 0 22 * * * % RUN_FULL_QA=true;DISABLE_DL_KERNELS=true;RUN_TILE_ENGINE_BASIC_TESTS=true;RUN_TILE_ENGINE_GEMM_TESTS=true;RUN_DISPATCHER_PERF_TESTS=true;RUN_DISPATCHER_CORRECTNESS_TESTS=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true 0 21 * * * % RUN_GROUPED_CONV_LARGE_CASES_TESTS=true;hipTensor_test=true;BUILD_GFX101=false;BUILD_GFX908=false;BUILD_GFX942=true;BUILD_GFX950=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true;BUILD_PACKAGES=true 0 19 * * * % BUILD_DOCKER=true;COMPILER_VERSION=develop;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true 0 17 * * * % BUILD_DOCKER=true;COMPILER_VERSION=therock;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true @@ -190,13 +190,13 @@ pipeline { defaultValue: false, description: "Run the tile_engine_gemm tests (default: OFF)") booleanParam( - name: "RUN_DISPATCHER_PERF_TESTS", + name: "RUN_DISPATCHER_CORRECTNESS_TESTS", defaultValue: true, - description: "Smoke-tier AOT benchmark on gfx942+gfx950: 64 instances/op, one problem size. gfx942: gemm_universal, batched_gemm, batched_contraction, gemm_streamk, grouped_gemm (fp16/rcr), gemm_multi_d, gemm_multi_abd (fp16/rcrr), grouped_gemm_rowcolquant, grouped_gemm_tensorquant (fp8/rcr). gfx950: gemm_aquant, gemm_bquant, gemm_abquant, mx_gemm (fp8/rcr), gemm_streamk (fp16/rcr). (default: OFF)") + description: "Run Correctness Tier for Dispatcher") booleanParam( - name: "RUN_DISPATCHER_CORRECTNESS_TESTS", + name: "RUN_DISPATCHER_PERF_TESTS", defaultValue: true, - description: "Smoke-tier JIT correctness on gfx942+gfx950: gemm_universal (budget 64), grouped_gemm, multi_d, multi_abd, batched_gemm, batched_contraction, stream_k, grouped_gemm_rowcolquant/tensorquant/aquant/abquant on both arches; mx_gemm + grouped_gemm_bquant on gfx950 only (ISA limitation). Does NOT cover preshuffled_gemm, block_scale_gemm ops, or batched_contraction_multi_abd (no bridge on develop). (default: OFF)") + description: "Run Performance Tier for Dispatcher") booleanParam( name: "BUILD_INSTANCES_ONLY", defaultValue: false, @@ -674,7 +674,7 @@ pipeline { } } } - stage("Run DISPATCHER_PERF Tests") + stage("Run TILE_ENGINE_GEMM Tests") { when { beforeAgent true @@ -682,11 +682,11 @@ pipeline { } parallel { - stage("Run DISPATCHER_PERF Tests on gfx942") + stage("Run TILE_ENGINE_GEMM Tests on gfx942") { when { beforeAgent true - expression { params.RUN_DISPATCHER_PERF_TESTS.toBoolean() } + expression { params.RUN_TILE_ENGINE_GEMM_TESTS.toBoolean() } } agent none steps{ @@ -694,17 +694,17 @@ pipeline { loadCk() ck.runOnHealthyNode(rocmnode("gfx942")) { deleteDir() - ck.runDispatcherPerfTests(params.BUILD_COMPILER, "gfx942") + ck.runTileEngineGemmTests("gfx942", params.BUILD_COMPILER) cleanWs() } } } } - stage("Run DISPATCHER_PERF Tests on gfx950") + stage("Run TILE_ENGINE_GEMM Tests on gfx950") { when { beforeAgent true - expression { params.RUN_DISPATCHER_PERF_TESTS.toBoolean() } + expression { params.RUN_TILE_ENGINE_GEMM_TESTS.toBoolean() } } agent none steps{ @@ -712,23 +712,13 @@ pipeline { loadCk() ck.runOnHealthyNode(rocmnode("gfx950")) { deleteDir() - ck.runDispatcherPerfTests(params.BUILD_COMPILER, "gfx950") + ck.runTileEngineGemmTests("gfx950", params.BUILD_COMPILER) cleanWs() } } } } - } - } - stage("Run TILE_ENGINE_GEMM Tests") - { - when { - beforeAgent true - expression { env.SHOULD_RUN_CI.toBoolean() } - } - parallel - { - stage("Run TILE_ENGINE_GEMM Tests on gfx942") + stage("Run TILE_ENGINE_GEMM Tests on gfx1201") { when { beforeAgent true @@ -738,45 +728,55 @@ pipeline { steps{ script { loadCk() - ck.runOnHealthyNode(rocmnode("gfx942")) { + ck.runOnHealthyNode(rocmnode("gfx1201")) { deleteDir() - ck.runTileEngineGemmTests("gfx942", params.BUILD_COMPILER) + ck.runTileEngineGemmTests("gfx1201", params.BUILD_COMPILER) cleanWs() } } } } - stage("Run TILE_ENGINE_GEMM Tests on gfx950") + } + } + stage("Run DISPATCHER_CORRECTNESS Tests") + { + when { + beforeAgent true + expression { env.SHOULD_RUN_CI.toBoolean() } + } + parallel + { + stage("Run DISPATCHER_CORRECTNESS Tests on gfx942") { when { beforeAgent true - expression { params.RUN_TILE_ENGINE_GEMM_TESTS.toBoolean() } + expression { params.RUN_DISPATCHER_CORRECTNESS_TESTS.toBoolean() } } agent none steps{ script { loadCk() - ck.runOnHealthyNode(rocmnode("gfx950")) { + ck.runOnHealthyNode(rocmnode("gfx942")) { deleteDir() - ck.runTileEngineGemmTests("gfx950", params.BUILD_COMPILER) + ck.runDispatcherCorrectnessTests("gfx942", params.BUILD_COMPILER) cleanWs() } } } } - stage("Run TILE_ENGINE_GEMM Tests on gfx1201") + stage("Run DISPATCHER_CORRECTNESS Tests on gfx950") { when { beforeAgent true - expression { params.RUN_TILE_ENGINE_GEMM_TESTS.toBoolean() } + expression { params.RUN_DISPATCHER_CORRECTNESS_TESTS.toBoolean() } } agent none steps{ script { loadCk() - ck.runOnHealthyNode(rocmnode("gfx1201")) { + ck.runOnHealthyNode(rocmnode("gfx950")) { deleteDir() - ck.runTileEngineGemmTests("gfx1201", params.BUILD_COMPILER) + ck.runDispatcherCorrectnessTests("gfx950", params.BUILD_COMPILER) cleanWs() } } @@ -784,12 +784,7 @@ pipeline { } } } - // The lane fans out to gfx942 and gfx950. The quant operators (mx_gemm, - // aquant, bquant, abquant) are gfx950-only, so covering the full operator - // surface is impossible from a single node type. Cost is held down by the - // search budget, not by dropping operators -- see the budget note on - // runDispatcherCorrectnessTests in groovy/vars/ck.groovy. - stage("Run DISPATCHER_CORRECTNESS Tests") + stage("Run DISPATCHER_PERF Tests") { when { beforeAgent true @@ -797,11 +792,11 @@ pipeline { } parallel { - stage("Run DISPATCHER_CORRECTNESS Tests on gfx942") + stage("Run DISPATCHER_PERF Tests on gfx942") { when { beforeAgent true - expression { params.RUN_DISPATCHER_CORRECTNESS_TESTS.toBoolean() } + expression { params.RUN_DISPATCHER_PERF_TESTS.toBoolean() } } agent none steps{ @@ -809,17 +804,17 @@ pipeline { loadCk() ck.runOnHealthyNode(rocmnode("gfx942")) { deleteDir() - ck.runDispatcherCorrectnessTests("gfx942", params.BUILD_COMPILER) + ck.runDispatcherPerfTests(params.BUILD_COMPILER, "gfx942") cleanWs() } } } } - stage("Run DISPATCHER_CORRECTNESS Tests on gfx950") + stage("Run DISPATCHER_PERF Tests on gfx950") { when { beforeAgent true - expression { params.RUN_DISPATCHER_CORRECTNESS_TESTS.toBoolean() } + expression { params.RUN_DISPATCHER_PERF_TESTS.toBoolean() } } agent none steps{ @@ -827,15 +822,12 @@ pipeline { loadCk() ck.runOnHealthyNode(rocmnode("gfx950")) { deleteDir() - ck.runDispatcherCorrectnessTests("gfx950", params.BUILD_COMPILER) + ck.runDispatcherPerfTests(params.BUILD_COMPILER, "gfx950") cleanWs() } } } } - // No gfx1201 dispatcher stage: the dispatcher has no RDNA4 tile - // coverage, so such a stage could only ever run red. See the - // arch note on dispatcherGemmVariantsFor in groovy/vars/ck.groovy. } } From bb6f1a348d8a5edebb0b236cd4e3cb58384dccc7 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Wed, 2 Sep 2026 16:45:30 -0400 Subject: [PATCH 42/51] refactor(ck-tile): merge dispatcher CI stages into one parallel block Collapse the two separate DISPATCHER_CORRECTNESS and DISPATCHER_PERF outer stages into a single "Run DISPATCHER Tests" stage with one parallel block containing all four arch sub-stages. Reduces the Jenkinsfile diff by 10 lines with no behavioral change. Co-Authored-By: Claude --- projects/composablekernel/Jenkinsfile | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index e70bb762dc48..a874915c11b5 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -738,7 +738,7 @@ pipeline { } } } - stage("Run DISPATCHER_CORRECTNESS Tests") + stage("Run DISPATCHER Tests") { when { beforeAgent true @@ -782,16 +782,6 @@ pipeline { } } } - } - } - stage("Run DISPATCHER_PERF Tests") - { - when { - beforeAgent true - expression { env.SHOULD_RUN_CI.toBoolean() } - } - parallel - { stage("Run DISPATCHER_PERF Tests on gfx942") { when { From d9cc5b334d1255414cce1b5801d5f1badbdacd95 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Wed, 2 Sep 2026 18:07:04 -0400 Subject: [PATCH 43/51] refactor(ck-tile): clean up dispatcher CI params and switch cases - Fix defaultValue: true -> false for RUN_DISPATCHER_CORRECTNESS_TESTS and RUN_DISPATCHER_PERF_TESTS; both params are opt-in and the PR description says they default OFF, matching every other lane param. - Collapse duplicate switch cases in dispatcherGemmVariantsFor and dispatcherSweepDtypesFor: gfx942 and gfx950 return identical values, so fall-through avoids repeating the return twice. Co-Authored-By: Claude --- projects/composablekernel/Jenkinsfile | 4 ++-- projects/composablekernel/groovy/vars/ck.groovy | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index a874915c11b5..6698bc25aef1 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -191,11 +191,11 @@ pipeline { description: "Run the tile_engine_gemm tests (default: OFF)") booleanParam( name: "RUN_DISPATCHER_CORRECTNESS_TESTS", - defaultValue: true, + defaultValue: false, description: "Run Correctness Tier for Dispatcher") booleanParam( name: "RUN_DISPATCHER_PERF_TESTS", - defaultValue: true, + defaultValue: false, description: "Run Performance Tier for Dispatcher") booleanParam( name: "BUILD_INSTANCES_ONLY", diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index e39f44a95089..85450798fd32 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1719,7 +1719,7 @@ def runDispatcherPerfTests(String compiler, String gpuTarget = "gfx942") { // configs to run". Covering RDNA4 needs real gfx1201 tile coverage first. def dispatcherGemmVariantsFor(String arch) { switch (arch) { - case "gfx942": return ["grouped", "multi_d", "multi_abd", "stream_k"] + case "gfx942": case "gfx950": return ["grouped", "multi_d", "multi_abd", "stream_k"] default: return [] } @@ -1746,7 +1746,7 @@ def dispatcherGemmVariantsFor(String arch) { // cannot drift apart again. def dispatcherSweepDtypesFor(String arch) { switch (arch) { - case "gfx942": return "fp16,bf16,fp8,bf8" + case "gfx942": case "gfx950": return "fp16,bf16,fp8,bf8" default: return "fp16,bf16" } From 1d4c188b58716f33488f237e1b30901da510ccfa Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Wed, 2 Sep 2026 18:42:21 -0400 Subject: [PATCH 44/51] fix(ck-tile): move dispatcher parallel stages to ck.groovy to fix MethodTooLargeException Jenkins CPS-compiles the entire declarative pipeline block into one JVM method. The 4-substage parallel block for the dispatcher lanes pushed the Jenkinsfile past the 64KB bytecode limit, causing: MethodTooLargeException: WorkflowScript.___cps___1950027 Fix: replace the 75-line declarative parallel block with a single scripted call to a new ck.runDispatcherTests() helper that builds the parallel map and runs it. The Jenkinsfile loses 74 lines of CPS-compiled DSL; ck.groovy gains 42 lines in the shared library where method size is not constrained the same way. Co-Authored-By: Claude --- projects/composablekernel/Jenkinsfile | 83 ++----------------- .../composablekernel/groovy/vars/ck.groovy | 42 ++++++++++ 2 files changed, 51 insertions(+), 74 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index 6698bc25aef1..5c2f0c3ec954 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -742,81 +742,16 @@ pipeline { { when { beforeAgent true - expression { env.SHOULD_RUN_CI.toBoolean() } + expression { env.SHOULD_RUN_CI.toBoolean() && (params.RUN_DISPATCHER_CORRECTNESS_TESTS.toBoolean() || params.RUN_DISPATCHER_PERF_TESTS.toBoolean()) } } - parallel - { - stage("Run DISPATCHER_CORRECTNESS Tests on gfx942") - { - when { - beforeAgent true - expression { params.RUN_DISPATCHER_CORRECTNESS_TESTS.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx942")) { - deleteDir() - ck.runDispatcherCorrectnessTests("gfx942", params.BUILD_COMPILER) - cleanWs() - } - } - } - } - stage("Run DISPATCHER_CORRECTNESS Tests on gfx950") - { - when { - beforeAgent true - expression { params.RUN_DISPATCHER_CORRECTNESS_TESTS.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx950")) { - deleteDir() - ck.runDispatcherCorrectnessTests("gfx950", params.BUILD_COMPILER) - cleanWs() - } - } - } - } - stage("Run DISPATCHER_PERF Tests on gfx942") - { - when { - beforeAgent true - expression { params.RUN_DISPATCHER_PERF_TESTS.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx942")) { - deleteDir() - ck.runDispatcherPerfTests(params.BUILD_COMPILER, "gfx942") - cleanWs() - } - } - } - } - stage("Run DISPATCHER_PERF Tests on gfx950") - { - when { - beforeAgent true - expression { params.RUN_DISPATCHER_PERF_TESTS.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx950")) { - deleteDir() - ck.runDispatcherPerfTests(params.BUILD_COMPILER, "gfx950") - cleanWs() - } - } - } + agent none + steps { + script { + loadCk() + ck.runDispatcherTests( + params.RUN_DISPATCHER_CORRECTNESS_TESTS.toBoolean(), + params.RUN_DISPATCHER_PERF_TESTS.toBoolean(), + params.BUILD_COMPILER) } } } diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 85450798fd32..acd3c27faca0 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1966,6 +1966,48 @@ def runDispatcherCorrectnessTests(String arch, String compiler) { } } +// Orchestrate correctness and perf dispatcher lanes in parallel. +// Called from the Jenkinsfile as a scripted step to keep the declarative +// pipeline method small enough to avoid the JVM 64KB method-size limit. +def runDispatcherTests(boolean runCorrectness, boolean runPerf, String compiler) { + def branches = [:] + if (runCorrectness) { + branches["DISPATCHER_CORRECTNESS gfx942"] = { + runOnHealthyNode(rocmnode("gfx942")) { + deleteDir() + runDispatcherCorrectnessTests("gfx942", compiler) + cleanWs() + } + } + branches["DISPATCHER_CORRECTNESS gfx950"] = { + runOnHealthyNode(rocmnode("gfx950")) { + deleteDir() + runDispatcherCorrectnessTests("gfx950", compiler) + cleanWs() + } + } + } + if (runPerf) { + branches["DISPATCHER_PERF gfx942"] = { + runOnHealthyNode(rocmnode("gfx942")) { + deleteDir() + runDispatcherPerfTests(compiler, "gfx942") + cleanWs() + } + } + branches["DISPATCHER_PERF gfx950"] = { + runOnHealthyNode(rocmnode("gfx950")) { + deleteDir() + runDispatcherPerfTests(compiler, "gfx950") + cleanWs() + } + } + } + if (branches) { + parallel branches + } +} + def runBuildCKAndTests(String arch) { def gpuTarget def extraSetupArgs = "" From ed95e4803911d6103530d98b613092be280a7b06 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Wed, 2 Sep 2026 19:45:43 -0400 Subject: [PATCH 45/51] fix(ck-tile): move downstream/FMHA/build-CK parallel stages to ck.groovy Jenkinsfile was still 1034 lines, triggering MethodTooLargeException on CPS compilation. Move three more large parallel blocks into ck.groovy functions: - runDownstreamTestsStages (Pytorch/AITER/FA) - runFMHATestsStages (gfx90a/gfx942/gfx950/gfx1201) - runBuildCKStages (all 8 gfx targets) Jenkinsfile drops from 1034 to 605 lines; ck.groovy grows by 163 lines. Co-Authored-By: Claude --- projects/composablekernel/Jenkinsfile | 463 +----------------- .../composablekernel/groovy/vars/ck.groovy | 163 ++++++ 2 files changed, 180 insertions(+), 446 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index 5c2f0c3ec954..faa04fff1930 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -91,7 +91,7 @@ def showCompilerInfo(boolean deferBinaryInfo = false, String dockerImage = "") { //launch develop branch daily jobs CRON_SETTINGS = BRANCH_NAME == "develop" ? '''0 23 * * * % RUN_FULL_QA=true;RUN_CK_TILE_FMHA_TESTS=true;RUN_PERFORMANCE_TESTS=true;FORCE_CI=true - 0 22 * * * % RUN_FULL_QA=true;DISABLE_DL_KERNELS=true;RUN_TILE_ENGINE_BASIC_TESTS=true;RUN_TILE_ENGINE_GEMM_TESTS=true;RUN_DISPATCHER_PERF_TESTS=true;RUN_DISPATCHER_CORRECTNESS_TESTS=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true + 0 22 * * * % RUN_FULL_QA=true;DISABLE_DL_KERNELS=true;RUN_DISPATCHER_PERF_TESTS=true;RUN_DISPATCHER_CORRECTNESS_TESTS=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true 0 21 * * * % RUN_GROUPED_CONV_LARGE_CASES_TESTS=true;hipTensor_test=true;BUILD_GFX101=false;BUILD_GFX908=false;BUILD_GFX942=true;BUILD_GFX950=true;RUN_PERFORMANCE_TESTS=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true;BUILD_PACKAGES=true 0 19 * * * % BUILD_DOCKER=true;COMPILER_VERSION=develop;BUILD_COMPILER=/llvm-project/build/bin/clang++;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true 0 17 * * * % BUILD_DOCKER=true;COMPILER_VERSION=therock;USE_SCCACHE=false;NINJA_BUILD_TRACE=true;RUN_ALL_UNIT_TESTS=true;FORCE_CI=true @@ -181,14 +181,6 @@ pipeline { name: "RUN_CK_TILE_FMHA_TESTS", defaultValue: false, description: "Run the ck_tile FMHA tests (default: OFF)") - booleanParam( - name: "RUN_TILE_ENGINE_BASIC_TESTS", - defaultValue: true, - description: "Run the tile_engine_basic tests (default: ON)") - booleanParam( - name: "RUN_TILE_ENGINE_GEMM_TESTS", - defaultValue: false, - description: "Run the tile_engine_gemm tests (default: OFF)") booleanParam( name: "RUN_DISPATCHER_CORRECTNESS_TESTS", defaultValue: false, @@ -391,92 +383,11 @@ pipeline { beforeAgent true expression { env.SHOULD_RUN_CI.toBoolean() } } - parallel - { - stage("Run Pytorch Tests on gfx942") - { - when { - beforeAgent true - expression { params.RUN_PYTORCH_TESTS.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx942")) { - ck.run_downstream_tests(image: "${env.CK_PYTORCH_IMAGE}", timeoutHours: 2, execute_cmds: ck.getPytorchTestsCmds()) - cleanWs() - } - } - } - } - stage("Run AITER Tests on gfx942") - { - when { - beforeAgent true - expression { params.RUN_AITER_TESTS.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx942")) { - ck.run_downstream_tests(image: "${env.CK_AITER_IMAGE}", timeoutHours: 5, execute_cmds: ck.getAiterTestsCmds()) - cleanWs() - } - } - } - } - stage("Run AITER Tests on gfx950") - { - when { - beforeAgent true - expression { params.RUN_AITER_TESTS.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx950")) { - ck.run_downstream_tests(image: "${env.CK_AITER_IMAGE}", timeoutHours: 5, execute_cmds: ck.getAiterTestsCmds()) - cleanWs() - } - } - } - } - stage("Run FA Tests on gfx942") - { - when { - beforeAgent true - expression { params.RUN_FA_TESTS.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx942")) { - ck.run_downstream_tests(image: "${env.CK_FA_IMAGE}", timeoutHours: 5, execute_cmds: ck.getFaTestsCmds()) - cleanWs() - } - } - } - } - stage("Run FA Tests on gfx950") - { - when { - beforeAgent true - expression { params.RUN_FA_TESTS.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx950")) { - ck.run_downstream_tests(image: "${env.CK_FA_IMAGE}", timeoutHours: 5, execute_cmds: ck.getFaTestsCmds()) - cleanWs() - } - } - } + agent none + steps { + script { + loadCk() + ck.runDownstreamTestsStages(this.&rocmnode, params) } } } @@ -570,171 +481,11 @@ pipeline { beforeAgent true expression { env.SHOULD_RUN_CI.toBoolean() } } - parallel - { - stage("Run CK_TILE_FMHA Tests on gfx90a") - { - when { - beforeAgent true - expression { params.RUN_CK_TILE_FMHA_TESTS.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx90a")) { - deleteDir() - ck.buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: ck.build_and_run_fmha("gfx90a")) - cleanWs() - } - } - } - } - stage("Run CK_TILE_FMHA Tests on gfx942") - { - when { - beforeAgent true - expression { params.RUN_CK_TILE_FMHA_TESTS.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx942")) { - deleteDir() - ck.buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: ck.build_and_run_fmha("gfx942")) - cleanWs() - } - } - } - } - stage("Run CK_TILE_FMHA Tests on gfx950") - { - when { - beforeAgent true - expression { params.RUN_CK_TILE_FMHA_TESTS.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx950")) { - deleteDir() - ck.buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: ck.build_and_run_fmha("gfx950")) - cleanWs() - } - } - } - } - stage("Run CK_TILE_FMHA Tests on gfx1201") - { - when { - beforeAgent true - expression { params.RUN_CK_TILE_FMHA_TESTS.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx1201")) { - deleteDir() - ck.buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: ck.build_and_run_fmha("gfx1201")) - cleanWs() - } - } - } - } - } - } - stage("Run TILE_ENGINE_BASIC Tests") - { - when { - beforeAgent true - expression { env.SHOULD_RUN_CI.toBoolean() } - } - parallel - { - stage("Run TILE_ENGINE_BASIC Tests on gfx942") - { - when { - beforeAgent true - expression { params.RUN_TILE_ENGINE_BASIC_TESTS.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx942")) { - deleteDir() - ck.runTileEngineBasicTests(params.BUILD_COMPILER) - cleanWs() - } - } - } - } - } - } - stage("Run TILE_ENGINE_GEMM Tests") - { - when { - beforeAgent true - expression { env.SHOULD_RUN_CI.toBoolean() } - } - parallel - { - stage("Run TILE_ENGINE_GEMM Tests on gfx942") - { - when { - beforeAgent true - expression { params.RUN_TILE_ENGINE_GEMM_TESTS.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx942")) { - deleteDir() - ck.runTileEngineGemmTests("gfx942", params.BUILD_COMPILER) - cleanWs() - } - } - } - } - stage("Run TILE_ENGINE_GEMM Tests on gfx950") - { - when { - beforeAgent true - expression { params.RUN_TILE_ENGINE_GEMM_TESTS.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx950")) { - deleteDir() - ck.runTileEngineGemmTests("gfx950", params.BUILD_COMPILER) - cleanWs() - } - } - } - } - stage("Run TILE_ENGINE_GEMM Tests on gfx1201") - { - when { - beforeAgent true - expression { params.RUN_TILE_ENGINE_GEMM_TESTS.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx1201")) { - deleteDir() - ck.runTileEngineGemmTests("gfx1201", params.BUILD_COMPILER) - cleanWs() - } - } - } + agent none + steps { + script { + loadCk() + ck.runFMHATestsStages(this.&rocmnode, params) } } } @@ -756,195 +507,17 @@ pipeline { } } - stage("Build CK and run Tests") + stage("Build CK and run Tests") { when { beforeAgent true expression { env.SHOULD_RUN_CI.toBoolean() } } - parallel - { - stage("Build CK and run Tests on gfx942") - { - when { - beforeAgent true - expression { (params.BUILD_GFX942.toBoolean() || params.RUN_FULL_QA.toBoolean()) && !params.BUILD_INSTANCES_ONLY.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx942")) { - deleteDir() - ck.runBuildCKAndTests("gfx942") - cleanWs() - } - } - } - } - stage("Build CK and run Tests on gfx950") - { - when { - beforeAgent true - expression { params.BUILD_GFX950.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx950")) { - deleteDir() - ck.runBuildCKAndTests("gfx950") - cleanWs() - } - } - } - } - /* - stage("Build CK and run Tests on gfx908") - { - when { - beforeAgent true - expression { params.BUILD_GFX908.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() } - } - agent{ label rocmnode("gfx908") } - steps{ - deleteDir() - script { loadCk(); ck.runBuildCKAndTests("gfx908") } - cleanWs() - } - } - */ - stage("Build CK and run Tests on gfx90a") - { - when { - beforeAgent true - expression { params.BUILD_GFX90A.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx90a")) { - deleteDir() - ck.runBuildCKAndTests("gfx90a") - cleanWs() - } - } - } - } - stage("Build CK instances for all supported targets") - { - when { - beforeAgent true - expression { params.BUILD_INSTANCES_ONLY.toBoolean() && !params.RUN_FULL_QA.toBoolean() } - } - agent none - environment{ - setup_args = "NO_CK_BUILD" - execute_args = """ cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ - -DCMAKE_CXX_COMPILER="${params.BUILD_COMPILER}" \ - -DCMAKE_HIP_COMPILER="${params.BUILD_COMPILER}" \ - -DGPU_ARCHS="gfx908;gfx90a;gfx942;gfx950;gfx10-3-generic;gfx11-generic;gfx12-generic" \ - -D CMAKE_BUILD_TYPE=Release .. && ninja -j64 """ - } - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx942")) { - deleteDir() - ck.runBuildInstancesOnly(params.BUILD_COMPILER) - cleanWs() - } - } - } - } - /* - stage("Build CK and run Tests on gfx1010") - { - when { - beforeAgent true - expression { params.BUILD_GFX101.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() } - } - agent{ label rocmnode("gfx1010") } - steps{ - deleteDir() - script { loadCk(); ck.runBuildCKAndTests("gfx10-1-generic") } - cleanWs() - } - } - */ - stage("Build CK and run Tests on gfx1030") - { - when { - beforeAgent true - expression { params.BUILD_GFX103.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx1030")) { - deleteDir() - ck.runBuildCKAndTests("gfx10-3-generic") - cleanWs() - } - } - } - } - stage("Build CK and run Tests on gfx11") - { - when { - beforeAgent true - expression { params.BUILD_GFX11.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode('miopen && (gfx1101 || gfx1100)') { - deleteDir() - ck.runBuildCKAndTests("gfx11-generic") - cleanWs() - } - } - } - } - stage("Build CK and run Tests on gfx1201") - { - when { - beforeAgent true - expression { params.BUILD_GFX12.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx1201")) { - deleteDir() - ck.runBuildCKAndTests("gfx12-generic") - cleanWs() - } - } - } - } - stage("Build CK for gfx1250") - { - when { - beforeAgent true - expression { params.BUILD_GFX1250.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean() } - } - agent none - steps{ - script { - loadCk() - ck.runOnHealthyNode(rocmnode("gfx90a")) { - deleteDir() - ck.runBuildCKAndTests("gfx1250") - cleanWs() - } - } - } + agent none + steps { + script { + loadCk() + ck.runBuildCKStages(this.&rocmnode, params) } } post { @@ -952,7 +525,6 @@ pipeline { node(rocmnode("nogpu")) { script { loadCk() - // Simulate capture ck.generateAndArchiveBuildTraceVisualization("ck_build_trace_gfx11.json") ck.generateAndArchiveBuildTraceVisualization("ck_build_trace_gfx12.json") ck.generateAndArchiveBuildTraceVisualization("ck_build_trace_gfx90a.json") @@ -966,7 +538,6 @@ pipeline { script { node(rocmnode("nogpu")) { loadCk() - // Report the parent stage build ck and run tests status ck.setGithubStatus("${env.STAGE_NAME}", 'success', "Stage ${env.STAGE_NAME} passed") echo "Reporting success status for build ck and run tests" } diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index acd3c27faca0..8f246e2b5c77 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -2059,3 +2059,166 @@ def runBuildInstancesOnly(String compiler) { -D CMAKE_BUILD_TYPE=Release .. && ninja -j${nthreads()}""" ) } + +def runDownstreamTestsStages(def rocmnode, def params) { + parallel([ + "Run Pytorch Tests on gfx942": { + if (params.RUN_PYTORCH_TESTS.toBoolean()) { + runOnHealthyNode(rocmnode("gfx942")) { + run_downstream_tests(image: "${env.CK_PYTORCH_IMAGE}", timeoutHours: 2, execute_cmds: getPytorchTestsCmds()) + cleanWs() + } + } + }, + "Run AITER Tests on gfx942": { + if (params.RUN_AITER_TESTS.toBoolean()) { + runOnHealthyNode(rocmnode("gfx942")) { + run_downstream_tests(image: "${env.CK_AITER_IMAGE}", timeoutHours: 5, execute_cmds: getAiterTestsCmds()) + cleanWs() + } + } + }, + "Run AITER Tests on gfx950": { + if (params.RUN_AITER_TESTS.toBoolean()) { + runOnHealthyNode(rocmnode("gfx950")) { + run_downstream_tests(image: "${env.CK_AITER_IMAGE}", timeoutHours: 5, execute_cmds: getAiterTestsCmds()) + cleanWs() + } + } + }, + "Run FA Tests on gfx942": { + if (params.RUN_FA_TESTS.toBoolean()) { + runOnHealthyNode(rocmnode("gfx942")) { + run_downstream_tests(image: "${env.CK_FA_IMAGE}", timeoutHours: 5, execute_cmds: getFaTestsCmds()) + cleanWs() + } + } + }, + "Run FA Tests on gfx950": { + if (params.RUN_FA_TESTS.toBoolean()) { + runOnHealthyNode(rocmnode("gfx950")) { + run_downstream_tests(image: "${env.CK_FA_IMAGE}", timeoutHours: 5, execute_cmds: getFaTestsCmds()) + cleanWs() + } + } + } + ]) +} + +def runFMHATestsStages(def rocmnode, def params) { + parallel([ + "Run CK_TILE_FMHA Tests on gfx90a": { + if (params.RUN_CK_TILE_FMHA_TESTS.toBoolean()) { + runOnHealthyNode(rocmnode("gfx90a")) { + deleteDir() + buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: build_and_run_fmha("gfx90a")) + cleanWs() + } + } + }, + "Run CK_TILE_FMHA Tests on gfx942": { + if (params.RUN_CK_TILE_FMHA_TESTS.toBoolean()) { + runOnHealthyNode(rocmnode("gfx942")) { + deleteDir() + buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: build_and_run_fmha("gfx942")) + cleanWs() + } + } + }, + "Run CK_TILE_FMHA Tests on gfx950": { + if (params.RUN_CK_TILE_FMHA_TESTS.toBoolean()) { + runOnHealthyNode(rocmnode("gfx950")) { + deleteDir() + buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: build_and_run_fmha("gfx950")) + cleanWs() + } + } + }, + "Run CK_TILE_FMHA Tests on gfx1201": { + if (params.RUN_CK_TILE_FMHA_TESTS.toBoolean()) { + runOnHealthyNode(rocmnode("gfx1201")) { + deleteDir() + buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: build_and_run_fmha("gfx1201")) + cleanWs() + } + } + } + ]) +} + +def runBuildCKStages(def rocmnode, def params) { + parallel([ + "Build CK and run Tests on gfx942": { + if ((params.BUILD_GFX942.toBoolean() || params.RUN_FULL_QA.toBoolean()) && !params.BUILD_INSTANCES_ONLY.toBoolean()) { + runOnHealthyNode(rocmnode("gfx942")) { + deleteDir() + runBuildCKAndTests("gfx942") + cleanWs() + } + } + }, + "Build CK and run Tests on gfx950": { + if (params.BUILD_GFX950.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean()) { + runOnHealthyNode(rocmnode("gfx950")) { + deleteDir() + runBuildCKAndTests("gfx950") + cleanWs() + } + } + }, + "Build CK and run Tests on gfx90a": { + if (params.BUILD_GFX90A.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean()) { + runOnHealthyNode(rocmnode("gfx90a")) { + deleteDir() + runBuildCKAndTests("gfx90a") + cleanWs() + } + } + }, + "Build CK instances for all supported targets": { + if (params.BUILD_INSTANCES_ONLY.toBoolean() && !params.RUN_FULL_QA.toBoolean()) { + runOnHealthyNode(rocmnode("gfx942")) { + deleteDir() + runBuildInstancesOnly(params.BUILD_COMPILER) + cleanWs() + } + } + }, + "Build CK and run Tests on gfx1030": { + if (params.BUILD_GFX103.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean()) { + runOnHealthyNode(rocmnode("gfx1030")) { + deleteDir() + runBuildCKAndTests("gfx10-3-generic") + cleanWs() + } + } + }, + "Build CK and run Tests on gfx11": { + if (params.BUILD_GFX11.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean()) { + runOnHealthyNode('miopen && (gfx1101 || gfx1100)') { + deleteDir() + runBuildCKAndTests("gfx11-generic") + cleanWs() + } + } + }, + "Build CK and run Tests on gfx1201": { + if (params.BUILD_GFX12.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean()) { + runOnHealthyNode(rocmnode("gfx1201")) { + deleteDir() + runBuildCKAndTests("gfx12-generic") + cleanWs() + } + } + }, + "Build CK for gfx1250": { + if (params.BUILD_GFX1250.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean()) { + runOnHealthyNode(rocmnode("gfx90a")) { + deleteDir() + runBuildCKAndTests("gfx1250") + cleanWs() + } + } + } + ]) +} From ced06615cab9d4dcb0969b4f4b92ea31bf4d5df2 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Tue, 8 Sep 2026 16:19:49 -0400 Subject: [PATCH 46/51] fix(ck-tile): pass rocmnode into runDispatcherTests, retire tile_engine lanes runDispatcherTests called rocmnode() directly, but rocmnode is defined in the Jenkinsfile and a vars/ script does not inherit Jenkinsfile-local methods. All four call sites would have thrown MissingMethodException before reaching runOnHealthyNode. Both params default to false so PR builds never hit it, but the 22:00 develop cron sets both true. Now passed as the first parameter, the same convention runDownstreamTestsStages / runFMHATestsStages / runBuildCKStages already use. Delete runTileEngineBasicTests and runTileEngineGemmTests. Their parameters and stages were removed earlier on this branch, leaving both helpers with no callers. Retiring them on purpose: the operators they covered (preshuffle, the block_scale_gemm quant variants, batched contraction multiple ABD) are the next-phase set and mostly have no dispatcher bridge on develop yet, so a lane for them can only run red. The coverage delta belongs in the PR description. Fix the perf lane archive glob. That lane drives tile_engine's own benchmark drivers, so every file it writes is a tile_engine JSON -- gemm_universal_results .json and friends. Nothing there carries a dispatcher_ prefix; only the correctness lane writes those. The narrower glob matched zero files and allowEmptyArchive turned that into a silent success. allowEmptyArchive stays true because the call is in a finally block: on a cmake/ninja failure no JSON exists yet and a throwing archive step would mask the real exception. The signal that the run produced results moves into the drivers instead. Co-Authored-By: Claude Opus 5 --- projects/composablekernel/Jenkinsfile | 1 + .../composablekernel/groovy/vars/ck.groovy | 156 +++--------------- 2 files changed, 24 insertions(+), 133 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index faa04fff1930..57e24a1d2554 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -500,6 +500,7 @@ pipeline { script { loadCk() ck.runDispatcherTests( + this.&rocmnode, params.RUN_DISPATCHER_CORRECTNESS_TESTS.toBoolean(), params.RUN_DISPATCHER_PERF_TESTS.toBoolean(), params.BUILD_COMPILER) diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 8f246e2b5c77..29cfbfb32769 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1404,135 +1404,6 @@ def runComprehensiveConvDatasetTests() { ) } -@NonCPS -String _tileEngineBasicCmd(String compiler) { - return """ - cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ - -D BUILD_CK_TILE_ENGINE="ON" \ - -D CMAKE_CXX_COMPILER="${compiler}" \ - -D CMAKE_BUILD_TYPE=Release \ - -D GPU_TARGETS="gfx942" \ - -D GEMM_UNIVERSAL_DATATYPE="fp8;fp16" \ - -D GEMM_UNIVERSAL_LAYOUT="rcr;rrr;crr;ccr" \ - -D GEMM_UNIVERSAL_CONFIG_FILE="default_ci_config.json" \ - -D GEMM_MULTI_D_DATATYPE="fp16" \ - -D GEMM_MULTI_D_LAYOUT="rcrr;rrrr;crrr;ccrr" \ - -D GEMM_MULTI_D_CONFIG_FILE="default_ci_config.json" \ - -D GEMM_PRESHUFFLE_DATATYPE="fp16;fp8;bf16;bf8" \ - -D GEMM_PRESHUFFLE_LAYOUT="rcr" \ - -D GEMM_PRESHUFFLE_CONFIG_FILE="default_ci_config.json" .. && \ - ninja -j\$(nproc) benchmark_gemm_universal_all benchmark_gemm_preshuffle_all benchmark_gemm_multi_d_all && \ - python3 ../tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_universal_results.json && \ - python3 ../tile_engine/ops/gemm/gemm_preshuffle/gemm_preshuffle_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_preshuffle_results.json && \ - python3 ../tile_engine/ops/gemm/gemm_multi_d/gemm_multi_d_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_multi_d_results.json""" -} - -def runTileEngineBasicTests(String compiler) { - buildAndTest( - setup_args: "NO_CK_BUILD", - build_type: 'Release', - execute_cmd: _tileEngineBasicCmd(compiler) - ) -} - -@NonCPS -String _tileEngineGemmCmd_gfx942(String compiler) { - return """ - cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ - -D BUILD_CK_TILE_ENGINE="ON" \ - -D CMAKE_CXX_COMPILER="${compiler}" \ - -D CMAKE_BUILD_TYPE=Release \ - -D GPU_TARGETS="gfx942" \ - -D GEMM_UNIVERSAL_DATATYPE="fp8;fp16;bf8;bf16" \ - -D GEMM_UNIVERSAL_LAYOUT="rcr;rrr;crr;ccr" \ - -D GEMM_STREAMK_DATATYPE="fp8;fp16" \ - -D GEMM_STREAMK_LAYOUT="rcr" \ - -D GEMM_MULTI_D_DATATYPE="fp16" \ - -D GEMM_MULTI_D_LAYOUT="rcrr;rrrr;crrr;ccrr" \ - -D GEMM_PRESHUFFLE_DATATYPE="fp16;fp8;bf16;bf8" \ - -D GEMM_PRESHUFFLE_LAYOUT="rcr" \ - -D GROUPED_GEMM_DATATYPE="fp8;fp16" \ - -D GROUPED_GEMM_LAYOUT="rcr;rrr;crr;ccr" \ - -D GEMM_MULTI_ABD_DATATYPE="fp16" \ - -D GEMM_MULTI_ABD_LAYOUT="rcrr" \ - -D BATCHED_CONTRACTION_DATATYPE="fp16" \ - -D BATCHED_CONTRACTION_LAYOUT="rcr" \ - -D GEMM_ROWCOLQUANT_DATATYPE="fp8;bf8" \ - -D GEMM_ROWCOLQUANT_LAYOUT="rcr" \ - -D GEMM_TENSOR_QUANT_DATATYPE="fp8;bf8" \ - -D GEMM_TENSOR_QUANT_LAYOUT="rcr" \ - -D GROUPED_GEMM_ROWCOLQUANT_DATATYPE="fp8;bf8" \ - -D GROUPED_GEMM_ROWCOLQUANT_LAYOUT="rcr" \ - -D GROUPED_GEMM_TENSORQUANT_DATATYPE="fp8;bf8" \ - -D GROUPED_GEMM_TENSORQUANT_LAYOUT="rcr" \ - -D BATCHED_GEMM_DATATYPE="fp16" \ - -D BATCHED_GEMM_LAYOUT="rcr" \ - -D TILE_ENGINE_SAMPLING_TIER=daily .. && \ - ninja -j\$(nproc) benchmark_gemm_universal_all benchmark_gemm_preshuffle_all benchmark_gemm_multi_d_all benchmark_gemm_streamk_all benchmark_grouped_gemm_all benchmark_gemm_multi_abd_all benchmark_batched_contraction_all benchmark_gemm_rowcolquant_all benchmark_gemm_tensor_quant_all benchmark_grouped_gemm_rowcolquant_all benchmark_grouped_gemm_tensorquant_all benchmark_batched_gemm_all && \ - python3 ../tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_universal_results.json && \ - python3 ../tile_engine/ops/gemm/gemm_preshuffle/gemm_preshuffle_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_preshuffle_results.json && \ - python3 ../tile_engine/ops/gemm/gemm_multi_d/gemm_multi_d_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_multi_d_results.json && \ - python3 ../tile_engine/ops/gemm/grouped_gemm/grouped_gemm_benchmark.py . --problem-sizes "1024,1024,1024" --group-counts 8 --warmup 5 --repeat 5 --verbose --json grouped_gemm_results.json && \ - python3 ../tile_engine/ops/gemm/gemm_multi_abd/gemm_multi_abd_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_multi_abd_results.json && \ - python3 ../tile_engine/ops/gemm/batched_contraction/batched_contraction_benchmark.py . --problem-configs "g=2;m=1024;n=1024;k=1024" --warmup 5 --repeat 5 --verbose --json batched_contraction_results.json && \ - python3 ../tile_engine/ops/gemm/block_scale_gemm/gemm_rowcolquant/gemm_rowcolquant_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_rowcolquant_results.json && \ - python3 ../tile_engine/ops/gemm/block_scale_gemm/gemm_tensor_quant/gemm_tensor_quant_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_tensor_quant_results.json && \ - python3 ../tile_engine/ops/gemm/grouped_gemm_quant/grouped_gemm_rowcolquant/grouped_gemm_rowcolquant_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json grouped_gemm_rowcolquant_results.json && \ - python3 ../tile_engine/ops/gemm/grouped_gemm_quant/grouped_gemm_tensorquant/grouped_gemm_tensorquant_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json grouped_gemm_tensorquant_results.json && \ - python3 ../tile_engine/ops/gemm/batched_gemm/batched_gemm_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json batched_gemm_results.json """ -} - -@NonCPS -String _tileEngineGemmCmd_gfx950(String compiler) { - return """ - cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ - -D BUILD_CK_TILE_ENGINE="ON" \ - -D CMAKE_CXX_COMPILER="${compiler}" \ - -D CMAKE_BUILD_TYPE=Release \ - -D GPU_TARGETS="gfx950" \ - -D GEMM_UNIVERSAL_DATATYPE="fp8;fp16" \ - -D GEMM_UNIVERSAL_LAYOUT="rcr;rrr;crr;ccr" \ - -D GEMM_MULTI_D_DATATYPE="fp16" \ - -D GEMM_MULTI_D_LAYOUT="rcrr;rrrr;crrr;ccrr" \ - -D GEMM_PRESHUFFLE_DATATYPE="fp16;fp8;bf16;bf8" \ - -D GEMM_PRESHUFFLE_LAYOUT="rcr" \ - -D MX_GEMM_DATATYPE="fp4;fp8" \ - -D MX_GEMM_LAYOUT="rcr" \ - -D TILE_ENGINE_SAMPLING_TIER=daily .. && \ - ninja -j\$(nproc) benchmark_gemm_universal_all benchmark_gemm_preshuffle_all benchmark_gemm_multi_d_all && \ - python3 ../tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_universal_results.json && \ - python3 ../tile_engine/ops/gemm/gemm_preshuffle/gemm_preshuffle_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_preshuffle_results.json && \ - python3 ../tile_engine/ops/gemm/gemm_multi_d/gemm_multi_d_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_multi_d_results.json && \ - python3 ../tile_engine/ops/gemm/mx_gemm/mx_gemm_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json mx_gemm_results.json """ -} - -@NonCPS -String _tileEngineGemmCmd_gfx1201(String compiler) { - return """ - cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ - -D BUILD_CK_TILE_ENGINE="ON" \ - -D CMAKE_CXX_COMPILER="${compiler}" \ - -D CMAKE_BUILD_TYPE=Release \ - -D GPU_TARGETS="gfx1201" \ - -D GEMM_UNIVERSAL_DATATYPE="fp16" \ - -D GEMM_UNIVERSAL_LAYOUT="rcr;rrr;crr;ccr" \ - -D TILE_ENGINE_SAMPLING_TIER=daily .. && \ - ninja -j\$(nproc) benchmark_gemm_universal_all && \ - python3 ../tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py . --problem-sizes "1024,1024,1024" --warmup 5 --repeat 5 --verbose --json gemm_universal_results.json""" -} - -def runTileEngineGemmTests(String arch, String compiler) { - def execute_cmd - if (arch == "gfx942") { - execute_cmd = _tileEngineGemmCmd_gfx942(compiler) - } else if (arch == "gfx950") { - execute_cmd = _tileEngineGemmCmd_gfx950(compiler) - } else if (arch == "gfx1201") { - execute_cmd = _tileEngineGemmCmd_gfx1201(compiler) - } - buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: execute_cmd) -} - @NonCPS String _dispatcherPerfCmd(String compiler, String gpuTarget, String samplingTier, String gemmDatatype, String gemmLayout, String multiLayout, @@ -1690,9 +1561,21 @@ def runDispatcherPerfTests(String compiler, String gpuTarget = "gfx942") { // reasoning as runDispatcherCorrectnessTests: execute_cmd runs from // projects/composablekernel/build while archiveArtifacts resolves against // the workspace root, so the build-relative path is spelled out in full. - // dispatcher_* prefix matches the correctness lane and avoids accidentally - // capturing tile_engine benchmark JSONs that share the same build dir. - archiveArtifacts artifacts: "projects/composablekernel/build/dispatcher_*_results.json", + // + // *_results.json, NOT dispatcher_*: this lane drives tile_engine's own + // benchmark drivers, and every file it produces is a tile_engine JSON + // (gemm_universal_results.json, gemm_aquant_results.json, ...). Nothing + // here is written with a dispatcher_ prefix -- only the correctness lane + // writes those -- so the narrower glob matched zero files and + // allowEmptyArchive turned that into a silent success. + // + // allowEmptyArchive stays true only because this is a finally block: on a + // cmake/ninja failure no JSON exists yet, and a throwing archive step + // would mask the real exception. The signal that the run actually + // produced results comes from the drivers, which now exit non-zero when + // no kernels were discovered or the result set is empty, and record + // launches_attempted/succeeded/failed in each JSON. + archiveArtifacts artifacts: "projects/composablekernel/build/*_results.json", allowEmptyArchive: true } } @@ -1969,7 +1852,14 @@ def runDispatcherCorrectnessTests(String arch, String compiler) { // Orchestrate correctness and perf dispatcher lanes in parallel. // Called from the Jenkinsfile as a scripted step to keep the declarative // pipeline method small enough to avoid the JVM 64KB method-size limit. -def runDispatcherTests(boolean runCorrectness, boolean runPerf, String compiler) { +// +// rocmnode is passed in rather than called directly: it is defined in the +// Jenkinsfile (`def rocmnode(name)`), and a vars/ script does not inherit +// Jenkinsfile-local methods, so calling it here throws MissingMethodException +// before runOnHealthyNode is ever reached. Same convention as +// runDownstreamTestsStages / runFMHATestsStages / runBuildCKStages; the caller +// passes `this.&rocmnode`. +def runDispatcherTests(def rocmnode, boolean runCorrectness, boolean runPerf, String compiler) { def branches = [:] if (runCorrectness) { branches["DISPATCHER_CORRECTNESS gfx942"] = { From 733b24bb153731aa41db4d13d891657413454e7e Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Tue, 8 Sep 2026 16:23:02 -0400 Subject: [PATCH 47/51] fix(ck-tile): make tile_engine benchmark drivers report launch outcomes Every driver returned 0 unconditionally. A run where the build directory held no kernels, or where every launch failed at runtime, exited clean and the CI lane went green with an empty or absent results JSON. Fixing this in the Jenkins lane alone would only paper over it for one caller, so the accounting moves into the drivers themselves. GemmBenchmark now counts launch_attempted and launch_failed around run_kernel; the seven standalone drivers that carry their own run_kernel loop count the same way. export_json takes an optional launch_counts and adds launches_attempted/succeeded/failed to the metadata when given, so the trend is recoverable from an archived artifact. Exit codes: no launches attempted -> 1 (no kernels discovered; the build never produced anything to run) no results collected -> 1 (everything that was attempted failed) some launches failed -> 0, with a WARNING line The partial-failure case is deliberately a warning. This sweep runs every generated config and individual ones legitimately fail to launch -- a tile shape the arch does not support, a workspace too large for the problem size. Making any single failure red trades a permanently-green lane for a permanently-red one, which is no more informative. Total failure is already caught by the empty-result check, and the per-run counts are in the JSON for anyone tracking the trend. The three out-of-scope positional callers of export_json (gemm_rowcolquant, gemm_tensor_quant, gemm_preshuffle) are unaffected: launch_counts is the last parameter and defaults to None. Co-Authored-By: Claude Opus 5 --- .../tile_engine/ops/common/benchmark_utils.py | 30 ++++++++++---- .../batched_contraction_benchmark.py | 31 ++++++++++++++ .../batched_gemm/batched_gemm_benchmark.py | 31 ++++++++++++++ .../gemm_abquant/gemm_abquant_benchmark.py | 31 ++++++++++++++ .../gemm_aquant/gemm_aquant_benchmark.py | 31 ++++++++++++++ .../gemm_bquant/gemm_bquant_benchmark.py | 31 ++++++++++++++ .../tile_engine/ops/gemm/gemm_benchmark.py | 5 +++ .../gemm_multi_abd_benchmark.py | 33 ++++++++++++++- .../gemm_multi_d/gemm_multi_d_benchmark.py | 33 ++++++++++++++- .../gemm_universal_benchmark.py | 33 ++++++++++++++- .../grouped_gemm/grouped_gemm_benchmark.py | 31 ++++++++++++++ .../grouped_gemm_rowcolquant_benchmark.py | 40 ++++++++++++++++++- .../grouped_gemm_tensorquant_benchmark.py | 37 ++++++++++++++++- .../ops/gemm/mx_gemm/mx_gemm_benchmark.py | 33 ++++++++++++++- .../gemm_streamk/gemm_streamk_benchmark.py | 31 ++++++++++++++ 15 files changed, 447 insertions(+), 14 deletions(-) diff --git a/projects/composablekernel/tile_engine/ops/common/benchmark_utils.py b/projects/composablekernel/tile_engine/ops/common/benchmark_utils.py index f94bc4a969ac..b8ec4cffe952 100644 --- a/projects/composablekernel/tile_engine/ops/common/benchmark_utils.py +++ b/projects/composablekernel/tile_engine/ops/common/benchmark_utils.py @@ -143,7 +143,11 @@ def export_best_kernels(best_kernels: Dict, filename: str, verbose: bool = False def export_json( - results: List[Dict], filename: str, best_kernels: Dict = None, verbose: bool = False + results: List[Dict], + filename: str, + best_kernels: Dict = None, + verbose: bool = False, + launch_counts: Dict = None, ): """Export all results and best kernels to JSON with comprehensive metadata""" from datetime import datetime @@ -222,14 +226,24 @@ def export_json( r.get("tflops", 0) for r in relevant_results ) / len(relevant_results) + benchmark_metadata = { + "timestamp": datetime.now().isoformat(), + "total_kernels_tested": len(results), + "unique_kernels": len(set(r.get("name", "unknown") for r in results)), + "successful_runs": len(successful_results), + "failed_runs": len(results) - len(successful_results), + } + + # Launch accounting, when the caller tracks it + if launch_counts is not None: + attempted = launch_counts.get("attempted", 0) + failed = launch_counts.get("failed", 0) + benchmark_metadata["launches_attempted"] = attempted + benchmark_metadata["launches_succeeded"] = attempted - failed + benchmark_metadata["launches_failed"] = failed + output_data = { - "benchmark_metadata": { - "timestamp": datetime.now().isoformat(), - "total_kernels_tested": len(results), - "unique_kernels": len(set(r.get("name", "unknown") for r in results)), - "successful_runs": len(successful_results), - "failed_runs": len(results) - len(successful_results), - }, + "benchmark_metadata": benchmark_metadata, "performance_summary": { "tflops_stats": { "best": max(tflops_values, default=0), diff --git a/projects/composablekernel/tile_engine/ops/gemm/batched_contraction/batched_contraction_benchmark.py b/projects/composablekernel/tile_engine/ops/gemm/batched_contraction/batched_contraction_benchmark.py index 599183016bff..f8f9c4817094 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/batched_contraction/batched_contraction_benchmark.py +++ b/projects/composablekernel/tile_engine/ops/gemm/batched_contraction/batched_contraction_benchmark.py @@ -16,6 +16,8 @@ def __init__(self, build_dir: str, verbose: bool = False): self.build_dir = Path(build_dir) self.verbose = verbose self.results = [] + self.launch_attempted = 0 + self.launch_failed = 0 def discover_kernels(self) -> List[Path]: """Find all benchmark_batched_contraction_* executables in the build directory""" @@ -243,6 +245,7 @@ def benchmark_problem_size( for kernel_path in kernels: kernel_info = self.extract_kernel_info(kernel_path) + self.launch_attempted += 1 result = self.run_kernel(kernel_path, params) if result: @@ -276,6 +279,8 @@ def benchmark_problem_size( f"{structured_result['bandwidth_gb_s']:.2f} GB/s, " f"{structured_result['time_ms']:.2f}ms" ) + else: + self.launch_failed += 1 return results @@ -464,6 +469,9 @@ def export_json(self, filename: str, best_kernels: Dict = None): ), "successful_runs": len(successful_results), "failed_runs": len(self.results) - len(successful_results), + "launches_attempted": self.launch_attempted, + "launches_succeeded": self.launch_attempted - self.launch_failed, + "launches_failed": self.launch_failed, }, "performance_summary": { "tflops_stats": { @@ -595,6 +603,29 @@ def main(): if args.json: benchmark.export_json(args.json, best_kernels) + # Exiting 0 after every launch failed would leave the CI lane green with no signal + attempted = benchmark.launch_attempted + failed = benchmark.launch_failed + succeeded = attempted - failed + print(f"Launches: {attempted} attempted, {succeeded} succeeded, {failed} failed") + + if attempted == 0: + print("No kernel launches were attempted - no kernels discovered") + return 1 + if not benchmark.results: + print("No benchmark results were collected") + return 1 + if failed > 0: + # A warning, not a failure: this sweep runs every generated config, and + # individual configs legitimately fail to launch (unsupported tile shape + # for the arch, workspace too large for the problem size). Making any + # single failure red would trade a permanently-green lane for a + # permanently-red one, which is no more informative. Total failure is + # already caught by the empty-result check above, and the per-run counts + # are in the JSON (launches_attempted/succeeded/failed) for anyone + # tracking the trend. + print(f"WARNING: {failed} of {attempted} kernel launch(es) failed") + return 0 diff --git a/projects/composablekernel/tile_engine/ops/gemm/batched_gemm/batched_gemm_benchmark.py b/projects/composablekernel/tile_engine/ops/gemm/batched_gemm/batched_gemm_benchmark.py index 8a67f56aad73..45acd8fffb08 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/batched_gemm/batched_gemm_benchmark.py +++ b/projects/composablekernel/tile_engine/ops/gemm/batched_gemm/batched_gemm_benchmark.py @@ -16,6 +16,8 @@ def __init__(self, build_dir: str, verbose: bool = False): self.build_dir = Path(build_dir) self.verbose = verbose self.results = [] + self.launch_attempted = 0 + self.launch_failed = 0 def discover_kernels(self) -> List[Path]: """Find all benchmark_batched_gemm_* executables in the build directory""" @@ -294,6 +296,7 @@ def benchmark_problem_size( for kernel_path in kernels: kernel_info = self.extract_kernel_info(kernel_path) + self.launch_attempted += 1 result = self.run_kernel(kernel_path, params) if result: @@ -327,6 +330,8 @@ def benchmark_problem_size( print( f" {kernel_info['config_id']}: {structured_result['tflops']:.2f} TFLOPS, {structured_result['bandwidth_gb_s']:.2f} GB/s, {structured_result['time_ms']:.2f}ms" ) + else: + self.launch_failed += 1 return results @@ -517,6 +522,9 @@ def export_json(self, filename: str, best_kernels: Dict = None): ), "successful_runs": len(successful_results), "failed_runs": len(self.results) - len(successful_results), + "launches_attempted": self.launch_attempted, + "launches_succeeded": self.launch_attempted - self.launch_failed, + "launches_failed": self.launch_failed, }, "performance_summary": { "tflops_stats": { @@ -669,6 +677,29 @@ def main(): if args.json: benchmark.export_json(args.json, best_kernels) + # Exiting 0 after every launch failed would leave the CI lane green with no signal + attempted = benchmark.launch_attempted + failed = benchmark.launch_failed + succeeded = attempted - failed + print(f"Launches: {attempted} attempted, {succeeded} succeeded, {failed} failed") + + if attempted == 0: + print("No kernel launches were attempted - no kernels discovered") + return 1 + if not benchmark.results: + print("No benchmark results were collected") + return 1 + if failed > 0: + # A warning, not a failure: this sweep runs every generated config, and + # individual configs legitimately fail to launch (unsupported tile shape + # for the arch, workspace too large for the problem size). Making any + # single failure red would trade a permanently-green lane for a + # permanently-red one, which is no more informative. Total failure is + # already caught by the empty-result check above, and the per-run counts + # are in the JSON (launches_attempted/succeeded/failed) for anyone + # tracking the trend. + print(f"WARNING: {failed} of {attempted} kernel launch(es) failed") + return 0 diff --git a/projects/composablekernel/tile_engine/ops/gemm/block_scale_gemm/gemm_abquant/gemm_abquant_benchmark.py b/projects/composablekernel/tile_engine/ops/gemm/block_scale_gemm/gemm_abquant/gemm_abquant_benchmark.py index 86422986464a..19e043d6f18d 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/block_scale_gemm/gemm_abquant/gemm_abquant_benchmark.py +++ b/projects/composablekernel/tile_engine/ops/gemm/block_scale_gemm/gemm_abquant/gemm_abquant_benchmark.py @@ -15,6 +15,8 @@ def __init__(self, build_dir: str, verbose: bool = False): self.build_dir = Path(build_dir) self.verbose = verbose self.results = [] + self.launch_attempted = 0 + self.launch_failed = 0 def discover_kernels(self) -> List[Path]: """Find all benchmark_gemm_abquant_* executables in the build directory.""" @@ -268,6 +270,7 @@ def benchmark_problem_size( "rotating_count": rotating_count, } + self.launch_attempted += 1 result = self.run_kernel(kernel_path, params) if result: @@ -302,6 +305,8 @@ def benchmark_problem_size( f"{structured_result['bandwidth_gb_s']:.2f} GB/s, " f"{structured_result['time_ms']:.2f}ms" ) + else: + self.launch_failed += 1 return results @@ -382,6 +387,9 @@ def export_json(self, filename: str, best_kernels: Dict = None): "successful_runs": len( [r for r in self.results if r.get("tflops", 0) > 0] ), + "launches_attempted": self.launch_attempted, + "launches_succeeded": self.launch_attempted - self.launch_failed, + "launches_failed": self.launch_failed, }, "kernel_results": self.results, "best_kernels_by_problem": best_kernels or {}, @@ -470,6 +478,29 @@ def main(): if args.json: benchmark.export_json(args.json, best_kernels) + # Exiting 0 after every launch failed would leave the CI lane green with no signal + attempted = benchmark.launch_attempted + failed = benchmark.launch_failed + succeeded = attempted - failed + print(f"Launches: {attempted} attempted, {succeeded} succeeded, {failed} failed") + + if attempted == 0: + print("No kernel launches were attempted - no kernels discovered") + return 1 + if not benchmark.results: + print("No benchmark results were collected") + return 1 + if failed > 0: + # A warning, not a failure: this sweep runs every generated config, and + # individual configs legitimately fail to launch (unsupported tile shape + # for the arch, workspace too large for the problem size). Making any + # single failure red would trade a permanently-green lane for a + # permanently-red one, which is no more informative. Total failure is + # already caught by the empty-result check above, and the per-run counts + # are in the JSON (launches_attempted/succeeded/failed) for anyone + # tracking the trend. + print(f"WARNING: {failed} of {attempted} kernel launch(es) failed") + return 0 diff --git a/projects/composablekernel/tile_engine/ops/gemm/block_scale_gemm/gemm_aquant/gemm_aquant_benchmark.py b/projects/composablekernel/tile_engine/ops/gemm/block_scale_gemm/gemm_aquant/gemm_aquant_benchmark.py index 2f5534064318..fb105ab40d63 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/block_scale_gemm/gemm_aquant/gemm_aquant_benchmark.py +++ b/projects/composablekernel/tile_engine/ops/gemm/block_scale_gemm/gemm_aquant/gemm_aquant_benchmark.py @@ -16,6 +16,8 @@ def __init__(self, build_dir: str, verbose: bool = False): self.build_dir = Path(build_dir) self.verbose = verbose self.results = [] + self.launch_attempted = 0 + self.launch_failed = 0 def discover_kernels(self) -> List[Path]: """Find all benchmark_gemm_aquant_* executables in the build directory.""" @@ -275,6 +277,7 @@ def benchmark_problem_size( for kernel_path in kernels: kernel_info = self.extract_kernel_info(kernel_path) + self.launch_attempted += 1 result = self.run_kernel(kernel_path, params) if result: @@ -309,6 +312,8 @@ def benchmark_problem_size( f"{structured_result['bandwidth_gb_s']:.2f} GB/s, " f"{structured_result['time_ms']:.2f}ms" ) + else: + self.launch_failed += 1 return results @@ -431,6 +436,9 @@ def export_json(self, filename: str, best_kernels: Dict = None): "successful_runs": len( [r for r in self.results if r.get("tflops", 0) > 0] ), + "launches_attempted": self.launch_attempted, + "launches_succeeded": self.launch_attempted - self.launch_failed, + "launches_failed": self.launch_failed, }, "kernel_results": self.results, "best_kernels_by_problem": best_kernels or {}, @@ -540,6 +548,29 @@ def main(): if args.json: benchmark.export_json(args.json, best_kernels) + # Exiting 0 after every launch failed would leave the CI lane green with no signal + attempted = benchmark.launch_attempted + failed = benchmark.launch_failed + succeeded = attempted - failed + print(f"Launches: {attempted} attempted, {succeeded} succeeded, {failed} failed") + + if attempted == 0: + print("No kernel launches were attempted - no kernels discovered") + return 1 + if not benchmark.results: + print("No benchmark results were collected") + return 1 + if failed > 0: + # A warning, not a failure: this sweep runs every generated config, and + # individual configs legitimately fail to launch (unsupported tile shape + # for the arch, workspace too large for the problem size). Making any + # single failure red would trade a permanently-green lane for a + # permanently-red one, which is no more informative. Total failure is + # already caught by the empty-result check above, and the per-run counts + # are in the JSON (launches_attempted/succeeded/failed) for anyone + # tracking the trend. + print(f"WARNING: {failed} of {attempted} kernel launch(es) failed") + return 0 diff --git a/projects/composablekernel/tile_engine/ops/gemm/block_scale_gemm/gemm_bquant/gemm_bquant_benchmark.py b/projects/composablekernel/tile_engine/ops/gemm/block_scale_gemm/gemm_bquant/gemm_bquant_benchmark.py index b19d3702e2fc..2cfe9f0a7b74 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/block_scale_gemm/gemm_bquant/gemm_bquant_benchmark.py +++ b/projects/composablekernel/tile_engine/ops/gemm/block_scale_gemm/gemm_bquant/gemm_bquant_benchmark.py @@ -15,6 +15,8 @@ def __init__(self, build_dir: str, verbose: bool = False): self.build_dir = Path(build_dir) self.verbose = verbose self.results = [] + self.launch_attempted = 0 + self.launch_failed = 0 def discover_kernels(self) -> List[Path]: """Find all benchmark_gemm_bquant_* executables in the build directory.""" @@ -244,6 +246,7 @@ def benchmark_problem_size( for kernel_path in kernels: kernel_info = self.extract_kernel_info(kernel_path) + self.launch_attempted += 1 result = self.run_kernel(kernel_path, params) if result: @@ -278,6 +281,8 @@ def benchmark_problem_size( f"{structured_result['bandwidth_gb_s']:.2f} GB/s, " f"{structured_result['time_ms']:.2f}ms" ) + else: + self.launch_failed += 1 return results @@ -363,6 +368,9 @@ def export_json(self, filename: str, best_kernels: Dict = None): "successful_runs": len( [r for r in self.results if r.get("tflops", 0) > 0] ), + "launches_attempted": self.launch_attempted, + "launches_succeeded": self.launch_attempted - self.launch_failed, + "launches_failed": self.launch_failed, }, "kernel_results": self.results, "best_kernels_by_problem": best_kernels or {}, @@ -457,6 +465,29 @@ def main(): if args.json: benchmark.export_json(args.json, best_kernels) + # Exiting 0 after every launch failed would leave the CI lane green with no signal + attempted = benchmark.launch_attempted + failed = benchmark.launch_failed + succeeded = attempted - failed + print(f"Launches: {attempted} attempted, {succeeded} succeeded, {failed} failed") + + if attempted == 0: + print("No kernel launches were attempted - no kernels discovered") + return 1 + if not benchmark.results: + print("No benchmark results were collected") + return 1 + if failed > 0: + # A warning, not a failure: this sweep runs every generated config, and + # individual configs legitimately fail to launch (unsupported tile shape + # for the arch, workspace too large for the problem size). Making any + # single failure red would trade a permanently-green lane for a + # permanently-red one, which is no more informative. Total failure is + # already caught by the empty-result check above, and the per-run counts + # are in the JSON (launches_attempted/succeeded/failed) for anyone + # tracking the trend. + print(f"WARNING: {failed} of {attempted} kernel launch(es) failed") + return 0 diff --git a/projects/composablekernel/tile_engine/ops/gemm/gemm_benchmark.py b/projects/composablekernel/tile_engine/ops/gemm/gemm_benchmark.py index b35390a1f98e..ce40410565bb 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/gemm_benchmark.py +++ b/projects/composablekernel/tile_engine/ops/gemm/gemm_benchmark.py @@ -36,6 +36,8 @@ def __init__( self.verbose = verbose self.results = [] self.name = name + self.launch_attempted = 0 + self.launch_failed = 0 def discover_kernels(self) -> List[Path]: """Find all benchmark_gemm_* executables in the build directory""" @@ -244,6 +246,7 @@ def benchmark_problem_size( for kernel_path in kernels: kernel_info = self.extract_kernel_info(kernel_path) + self.launch_attempted += 1 result = benchmark_utils.run_kernel( self.build_dir, kernel_path, params, verbose=self.verbose ) @@ -278,6 +281,8 @@ def benchmark_problem_size( print( f" {kernel_info['config_id']}: {structured_result['tflops']:.2f} TFLOPS, {structured_result['bandwidth_gb_s']:.2f} GB/s, {structured_result['time_ms']:.2f}ms" ) + else: + self.launch_failed += 1 return results diff --git a/projects/composablekernel/tile_engine/ops/gemm/gemm_multi_abd/gemm_multi_abd_benchmark.py b/projects/composablekernel/tile_engine/ops/gemm/gemm_multi_abd/gemm_multi_abd_benchmark.py index 001e4a1d4f2b..82187ceb36db 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/gemm_multi_abd/gemm_multi_abd_benchmark.py +++ b/projects/composablekernel/tile_engine/ops/gemm/gemm_multi_abd/gemm_multi_abd_benchmark.py @@ -148,7 +148,38 @@ def main(): # Export JSON if requested if args.json: - benchmark_utils.export_json(benchmark.results, args.json, best_kernels) + benchmark_utils.export_json( + benchmark.results, + args.json, + best_kernels, + launch_counts={ + "attempted": benchmark.launch_attempted, + "failed": benchmark.launch_failed, + }, + ) + + # Exiting 0 after every launch failed would leave the CI lane green with no signal + attempted = benchmark.launch_attempted + failed = benchmark.launch_failed + succeeded = attempted - failed + print(f"Launches: {attempted} attempted, {succeeded} succeeded, {failed} failed") + + if attempted == 0: + print("No kernel launches were attempted - no kernels discovered") + return 1 + if not benchmark.results: + print("No benchmark results were collected") + return 1 + if failed > 0: + # A warning, not a failure: this sweep runs every generated config, and + # individual configs legitimately fail to launch (unsupported tile shape + # for the arch, workspace too large for the problem size). Making any + # single failure red would trade a permanently-green lane for a + # permanently-red one, which is no more informative. Total failure is + # already caught by the empty-result check above, and the per-run counts + # are in the JSON (launches_attempted/succeeded/failed) for anyone + # tracking the trend. + print(f"WARNING: {failed} of {attempted} kernel launch(es) failed") return 0 diff --git a/projects/composablekernel/tile_engine/ops/gemm/gemm_multi_d/gemm_multi_d_benchmark.py b/projects/composablekernel/tile_engine/ops/gemm/gemm_multi_d/gemm_multi_d_benchmark.py index 519644183776..7472a2e80a21 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/gemm_multi_d/gemm_multi_d_benchmark.py +++ b/projects/composablekernel/tile_engine/ops/gemm/gemm_multi_d/gemm_multi_d_benchmark.py @@ -140,7 +140,38 @@ def main(): # Export JSON if requested if args.json: - benchmark_utils.export_json(benchmark.results, args.json, best_kernels) + benchmark_utils.export_json( + benchmark.results, + args.json, + best_kernels, + launch_counts={ + "attempted": benchmark.launch_attempted, + "failed": benchmark.launch_failed, + }, + ) + + # Exiting 0 after every launch failed would leave the CI lane green with no signal + attempted = benchmark.launch_attempted + failed = benchmark.launch_failed + succeeded = attempted - failed + print(f"Launches: {attempted} attempted, {succeeded} succeeded, {failed} failed") + + if attempted == 0: + print("No kernel launches were attempted - no kernels discovered") + return 1 + if not benchmark.results: + print("No benchmark results were collected") + return 1 + if failed > 0: + # A warning, not a failure: this sweep runs every generated config, and + # individual configs legitimately fail to launch (unsupported tile shape + # for the arch, workspace too large for the problem size). Making any + # single failure red would trade a permanently-green lane for a + # permanently-red one, which is no more informative. Total failure is + # already caught by the empty-result check above, and the per-run counts + # are in the JSON (launches_attempted/succeeded/failed) for anyone + # tracking the trend. + print(f"WARNING: {failed} of {attempted} kernel launch(es) failed") return 0 diff --git a/projects/composablekernel/tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py b/projects/composablekernel/tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py index 73ba1261a849..6fe1b07633ed 100755 --- a/projects/composablekernel/tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py +++ b/projects/composablekernel/tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py @@ -140,7 +140,38 @@ def main(): # Export JSON if requested if args.json: - benchmark_utils.export_json(benchmark.results, args.json, best_kernels) + benchmark_utils.export_json( + benchmark.results, + args.json, + best_kernels, + launch_counts={ + "attempted": benchmark.launch_attempted, + "failed": benchmark.launch_failed, + }, + ) + + # Exiting 0 after every launch failed would leave the CI lane green with no signal + attempted = benchmark.launch_attempted + failed = benchmark.launch_failed + succeeded = attempted - failed + print(f"Launches: {attempted} attempted, {succeeded} succeeded, {failed} failed") + + if attempted == 0: + print("No kernel launches were attempted - no kernels discovered") + return 1 + if not benchmark.results: + print("No benchmark results were collected") + return 1 + if failed > 0: + # A warning, not a failure: this sweep runs every generated config, and + # individual configs legitimately fail to launch (unsupported tile shape + # for the arch, workspace too large for the problem size). Making any + # single failure red would trade a permanently-green lane for a + # permanently-red one, which is no more informative. Total failure is + # already caught by the empty-result check above, and the per-run counts + # are in the JSON (launches_attempted/succeeded/failed) for anyone + # tracking the trend. + print(f"WARNING: {failed} of {attempted} kernel launch(es) failed") return 0 diff --git a/projects/composablekernel/tile_engine/ops/gemm/grouped_gemm/grouped_gemm_benchmark.py b/projects/composablekernel/tile_engine/ops/gemm/grouped_gemm/grouped_gemm_benchmark.py index a9c6e167fedd..f446366ea8e4 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/grouped_gemm/grouped_gemm_benchmark.py +++ b/projects/composablekernel/tile_engine/ops/gemm/grouped_gemm/grouped_gemm_benchmark.py @@ -16,6 +16,8 @@ def __init__(self, build_dir: str, verbose: bool = False): self.build_dir = Path(build_dir) self.verbose = verbose self.results = [] + self.launch_attempted = 0 + self.launch_failed = 0 def discover_kernels(self) -> List[Path]: """Find all benchmark_grouped_gemm_* executables in the build directory""" @@ -308,6 +310,7 @@ def benchmark_problem_size( for kernel_path in kernels: kernel_info = self.extract_kernel_info(kernel_path) + self.launch_attempted += 1 result = self.run_kernel(kernel_path, params) if result: @@ -341,6 +344,8 @@ def benchmark_problem_size( print( f" {kernel_info['config_id']}: {structured_result['tflops']:.2f} TFLOPS, {structured_result['bandwidth_gb_s']:.2f} GB/s, {structured_result['time_ms']:.2f}ms" ) + else: + self.launch_failed += 1 return results @@ -534,6 +539,9 @@ def export_json(self, filename: str, best_kernels: Dict = None): ), "successful_runs": len(successful_results), "failed_runs": len(self.results) - len(successful_results), + "launches_attempted": self.launch_attempted, + "launches_succeeded": self.launch_attempted - self.launch_failed, + "launches_failed": self.launch_failed, }, "performance_summary": { "tflops_stats": { @@ -696,6 +704,29 @@ def main(): if args.json: benchmark.export_json(args.json, best_kernels) + # Exiting 0 after every launch failed would leave the CI lane green with no signal + attempted = benchmark.launch_attempted + failed = benchmark.launch_failed + succeeded = attempted - failed + print(f"Launches: {attempted} attempted, {succeeded} succeeded, {failed} failed") + + if attempted == 0: + print("No kernel launches were attempted - no kernels discovered") + return 1 + if not benchmark.results: + print("No benchmark results were collected") + return 1 + if failed > 0: + # A warning, not a failure: this sweep runs every generated config, and + # individual configs legitimately fail to launch (unsupported tile shape + # for the arch, workspace too large for the problem size). Making any + # single failure red would trade a permanently-green lane for a + # permanently-red one, which is no more informative. Total failure is + # already caught by the empty-result check above, and the per-run counts + # are in the JSON (launches_attempted/succeeded/failed) for anyone + # tracking the trend. + print(f"WARNING: {failed} of {attempted} kernel launch(es) failed") + return 0 diff --git a/projects/composablekernel/tile_engine/ops/gemm/grouped_gemm_quant/grouped_gemm_rowcolquant/grouped_gemm_rowcolquant_benchmark.py b/projects/composablekernel/tile_engine/ops/gemm/grouped_gemm_quant/grouped_gemm_rowcolquant/grouped_gemm_rowcolquant_benchmark.py index 6a4db06cd225..49c094e4b705 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/grouped_gemm_quant/grouped_gemm_rowcolquant/grouped_gemm_rowcolquant_benchmark.py +++ b/projects/composablekernel/tile_engine/ops/gemm/grouped_gemm_quant/grouped_gemm_rowcolquant/grouped_gemm_rowcolquant_benchmark.py @@ -88,6 +88,7 @@ def benchmark_problem_size( for kernel_path in kernels: kernel_info = self.extract_kernel_info(kernel_path) + self.launch_attempted += 1 result = benchmark_utils.run_kernel( self.build_dir, kernel_path, params, verbose=self.verbose ) @@ -122,6 +123,8 @@ def benchmark_problem_size( f" {kernel_info['config_id']}: {structured_result['tflops']:.2f} TFLOPS, " f"{structured_result['bandwidth_gb_s']:.2f} GB/s, {structured_result['time_ms']:.2f}ms" ) + else: + self.launch_failed += 1 return results @@ -301,6 +304,9 @@ def export_json(self, filename: str, best_kernels: Dict = None): ), "successful_runs": len(successful_results), "failed_runs": len(self.results) - len(successful_results), + "launches_attempted": self.launch_attempted, + "launches_succeeded": self.launch_attempted - self.launch_failed, + "launches_failed": self.launch_failed, }, "performance_summary": { "tflops_stats": { @@ -474,9 +480,41 @@ def main(): benchmark_utils.export_csv(benchmark.results, args.csv) benchmark_utils.export_best_kernels(best_kernels, args.best) if args.json: - benchmark_utils.export_json(benchmark.results, args.json, best_kernels) + benchmark_utils.export_json( + benchmark.results, + args.json, + best_kernels, + launch_counts={ + "attempted": benchmark.launch_attempted, + "failed": benchmark.launch_failed, + }, + ) print("\nBenchmark complete!") + + # Exiting 0 after every launch failed would leave the CI lane green with no signal + attempted = benchmark.launch_attempted + failed = benchmark.launch_failed + succeeded = attempted - failed + print(f"Launches: {attempted} attempted, {succeeded} succeeded, {failed} failed") + + if attempted == 0: + print("No kernel launches were attempted - no kernels discovered") + return 1 + if not benchmark.results: + print("No benchmark results were collected") + return 1 + if failed > 0: + # A warning, not a failure: this sweep runs every generated config, and + # individual configs legitimately fail to launch (unsupported tile shape + # for the arch, workspace too large for the problem size). Making any + # single failure red would trade a permanently-green lane for a + # permanently-red one, which is no more informative. Total failure is + # already caught by the empty-result check above, and the per-run counts + # are in the JSON (launches_attempted/succeeded/failed) for anyone + # tracking the trend. + print(f"WARNING: {failed} of {attempted} kernel launch(es) failed") + return 0 diff --git a/projects/composablekernel/tile_engine/ops/gemm/grouped_gemm_quant/grouped_gemm_tensorquant/grouped_gemm_tensorquant_benchmark.py b/projects/composablekernel/tile_engine/ops/gemm/grouped_gemm_quant/grouped_gemm_tensorquant/grouped_gemm_tensorquant_benchmark.py index 2d3557454e0e..e638d0c11388 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/grouped_gemm_quant/grouped_gemm_tensorquant/grouped_gemm_tensorquant_benchmark.py +++ b/projects/composablekernel/tile_engine/ops/gemm/grouped_gemm_quant/grouped_gemm_tensorquant/grouped_gemm_tensorquant_benchmark.py @@ -86,6 +86,7 @@ def benchmark_problem_size( for kernel_path in kernels: kernel_info = self.extract_kernel_info(kernel_path) + self.launch_attempted += 1 result = benchmark_utils.run_kernel( self.build_dir, kernel_path, params, verbose=self.verbose ) @@ -120,6 +121,8 @@ def benchmark_problem_size( f" {kernel_info['config_id']}: {structured_result['tflops']:.2f} TFLOPS, " f"{structured_result['bandwidth_gb_s']:.2f} GB/s, {structured_result['time_ms']:.2f}ms" ) + else: + self.launch_failed += 1 return results @@ -291,9 +294,41 @@ def main(): benchmark_utils.export_csv(benchmark.results, args.csv) benchmark_utils.export_best_kernels(best_kernels, args.best) if args.json: - benchmark_utils.export_json(benchmark.results, args.json, best_kernels) + benchmark_utils.export_json( + benchmark.results, + args.json, + best_kernels, + launch_counts={ + "attempted": benchmark.launch_attempted, + "failed": benchmark.launch_failed, + }, + ) print("\nBenchmark complete!") + + # Exiting 0 after every launch failed would leave the CI lane green with no signal + attempted = benchmark.launch_attempted + failed = benchmark.launch_failed + succeeded = attempted - failed + print(f"Launches: {attempted} attempted, {succeeded} succeeded, {failed} failed") + + if attempted == 0: + print("No kernel launches were attempted - no kernels discovered") + return 1 + if not benchmark.results: + print("No benchmark results were collected") + return 1 + if failed > 0: + # A warning, not a failure: this sweep runs every generated config, and + # individual configs legitimately fail to launch (unsupported tile shape + # for the arch, workspace too large for the problem size). Making any + # single failure red would trade a permanently-green lane for a + # permanently-red one, which is no more informative. Total failure is + # already caught by the empty-result check above, and the per-run counts + # are in the JSON (launches_attempted/succeeded/failed) for anyone + # tracking the trend. + print(f"WARNING: {failed} of {attempted} kernel launch(es) failed") + return 0 diff --git a/projects/composablekernel/tile_engine/ops/gemm/mx_gemm/mx_gemm_benchmark.py b/projects/composablekernel/tile_engine/ops/gemm/mx_gemm/mx_gemm_benchmark.py index f69bc8328688..8d64ef8bae7d 100644 --- a/projects/composablekernel/tile_engine/ops/gemm/mx_gemm/mx_gemm_benchmark.py +++ b/projects/composablekernel/tile_engine/ops/gemm/mx_gemm/mx_gemm_benchmark.py @@ -181,7 +181,38 @@ def main(): benchmark_utils.export_best_kernels(best_kernels, args.best) if args.json: - benchmark_utils.export_json(benchmark.results, args.json, best_kernels) + benchmark_utils.export_json( + benchmark.results, + args.json, + best_kernels, + launch_counts={ + "attempted": benchmark.launch_attempted, + "failed": benchmark.launch_failed, + }, + ) + + # Exiting 0 after every launch failed would leave the CI lane green with no signal + attempted = benchmark.launch_attempted + failed = benchmark.launch_failed + succeeded = attempted - failed + print(f"Launches: {attempted} attempted, {succeeded} succeeded, {failed} failed") + + if attempted == 0: + print("No kernel launches were attempted - no kernels discovered") + return 1 + if not benchmark.results: + print("No benchmark results were collected") + return 1 + if failed > 0: + # A warning, not a failure: this sweep runs every generated config, and + # individual configs legitimately fail to launch (unsupported tile shape + # for the arch, workspace too large for the problem size). Making any + # single failure red would trade a permanently-green lane for a + # permanently-red one, which is no more informative. Total failure is + # already caught by the empty-result check above, and the per-run counts + # are in the JSON (launches_attempted/succeeded/failed) for anyone + # tracking the trend. + print(f"WARNING: {failed} of {attempted} kernel launch(es) failed") return 0 diff --git a/projects/composablekernel/tile_engine/ops/gemm_streamk/gemm_streamk_benchmark.py b/projects/composablekernel/tile_engine/ops/gemm_streamk/gemm_streamk_benchmark.py index ad8d9ff35c5a..7c83d4f1fae5 100644 --- a/projects/composablekernel/tile_engine/ops/gemm_streamk/gemm_streamk_benchmark.py +++ b/projects/composablekernel/tile_engine/ops/gemm_streamk/gemm_streamk_benchmark.py @@ -16,6 +16,8 @@ def __init__(self, build_dir: str, verbose: bool = False): self.build_dir = Path(build_dir) self.verbose = verbose self.results = [] + self.launch_attempted = 0 + self.launch_failed = 0 def discover_kernels(self) -> List[Path]: """Find all benchmark_gemm_streamk_* executables in the build directory""" @@ -307,6 +309,7 @@ def benchmark_problem_size( for kernel_path in kernels: kernel_info = self.extract_kernel_info(kernel_path) + self.launch_attempted += 1 result = self.run_kernel(kernel_path, params) if result: @@ -341,6 +344,8 @@ def benchmark_problem_size( print( f" {kernel_info['config_id']}: {structured_result['tflops']:.2f} TFLOPS, {structured_result['bandwidth_gb_s']:.2f} GB/s, {structured_result['time_ms']:.2f}ms" ) + else: + self.launch_failed += 1 return results @@ -528,6 +533,9 @@ def export_json(self, filename: str, best_kernels: Dict = None): ), "successful_runs": len(successful_results), "failed_runs": len(self.results) - len(successful_results), + "launches_attempted": self.launch_attempted, + "launches_succeeded": self.launch_attempted - self.launch_failed, + "launches_failed": self.launch_failed, }, "performance_summary": { "tflops_stats": { @@ -669,6 +677,29 @@ def main(): if args.json: benchmark.export_json(args.json, best_kernels) + # Exiting 0 after every launch failed would leave the CI lane green with no signal + attempted = benchmark.launch_attempted + failed = benchmark.launch_failed + succeeded = attempted - failed + print(f"Launches: {attempted} attempted, {succeeded} succeeded, {failed} failed") + + if attempted == 0: + print("No kernel launches were attempted - no kernels discovered") + return 1 + if not benchmark.results: + print("No benchmark results were collected") + return 1 + if failed > 0: + # A warning, not a failure: this sweep runs every generated config, and + # individual configs legitimately fail to launch (unsupported tile shape + # for the arch, workspace too large for the problem size). Making any + # single failure red would trade a permanently-green lane for a + # permanently-red one, which is no more informative. Total failure is + # already caught by the empty-result check above, and the per-run counts + # are in the JSON (launches_attempted/succeeded/failed) for anyone + # tracking the trend. + print(f"WARNING: {failed} of {attempted} kernel launch(es) failed") + return 0 From 85a8cd3e41669183cfc5bd5eb7e30b36a984cc27 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Tue, 8 Sep 2026 16:23:20 -0400 Subject: [PATCH 48/51] fix(ck-tile): fix dispatcher sampler, reach the exit-77 skip path, stop false-green ctests Four related test-side defects, all of which made a lane report success without having exercised what it claims to cover. _sample handed out the divmod remainder in sorted stratum order. With budget < n_strata every per-stratum share is 0, so the entire budget was remainder and always landed on the first four keys -- bf16/{ccr,crr,rcr,rrr}. The ctest registration used --budget 4 against 16 strata, so no run in CI ever compiled an fp8 or bf8 config, and the daily rotating seed changed only which config was drawn inside those four fixed strata. Shuffle the stratum order before the remainder loop, and raise the ctest budget to 16 so every stratum is represented at least once. The skip path called detect_gpu_arch() bare. That helper defaults to fallback="gfx942" and so never returns falsy, making the `if not arch: return _SKIP` branch dead -- a CPU-only runner walked past it and compiled gfx942 kernels for hardware it does not have. Ask for fallback="" so the branch is reachable. The three unittest.TestCase suites (grouped_gemm / multi_abd / mx_gemm GPU correctness) were registered as bare `unittest discover`, which exits 0 when every test skips. mx_gemm is gfx950-only, so on gfx942 that is a green tick backed by zero assertions. They now run through run_unittest_77.py, which maps all-skipped onto 77 and pairs with SKIP_RETURN_CODE. A driver rather than converting the suites, so the skip decision stays in each test's own setUp. Discovery collecting nothing returns 1, not 77, so a renamed file cannot report Skipped forever. Add test_search_space_sampling.py pinning both the sampler and the skip path. CPU-only -- no GPU, hipcc or built kernels -- and both key tests were mutation checked against the old implementations to confirm they fail there. The sampler test asserts on stratum keys rather than whole samples because the buggy version did vary the configs within its four fixed strata; the skip test stubs detect_gpu_arch to honour the fallback it is handed, so it only passes if the caller actually asks for fallback="". Rename the quant helper functions from test_* to case_*. They take non-fixture positional args, so pytest collected them as tests and errored at fixture resolution -- 12 collection errors across the repo, now 0 with 1075 tests collected. Co-Authored-By: Claude Opus 5 --- .../dispatcher/tests/CMakeLists.txt | 50 ++++-- .../dispatcher/tests/run_unittest_77.py | 71 ++++++++ .../tests/test_abquant_gpu_correctness.py | 12 +- .../tests/test_aquant_gpu_correctness.py | 12 +- .../tests/test_bquant_gpu_correctness.py | 26 +-- .../tests/test_gemm_search_space.py | 29 +++- .../tests/test_search_space_sampling.py | 153 ++++++++++++++++++ 7 files changed, 313 insertions(+), 40 deletions(-) create mode 100644 projects/composablekernel/dispatcher/tests/run_unittest_77.py create mode 100644 projects/composablekernel/dispatcher/tests/test_search_space_sampling.py diff --git a/projects/composablekernel/dispatcher/tests/CMakeLists.txt b/projects/composablekernel/dispatcher/tests/CMakeLists.txt index 6c2edf42fba6..2d563d0033b3 100644 --- a/projects/composablekernel/dispatcher/tests/CMakeLists.txt +++ b/projects/composablekernel/dispatcher/tests/CMakeLists.txt @@ -174,9 +174,13 @@ set_tests_properties(dispatcher_test_gemm_utils PROPERTIES # when no GPU / hipcc / static lib is present, so they cost nothing on a # CPU-only runner and only run on a GPU runner. mx arch-gates to gfx950. # -# These three are still unittest.TestCase-based and skip internally, so -# `unittest discover` is the right driver. multi_d has been converted to the -# script-style-77 convention and is registered separately below. +# These three are unittest.TestCase-based and skip themselves in setUp, so they +# are driven through run_unittest_77.py rather than `python -m unittest +# discover` directly. discover exits 0 when every test skipped, which made ctest +# report Passed for a run that asserted nothing -- a CPU-only runner showed the +# same green tick as a real GPU run. The driver keeps the discovery semantics and +# only maps an all-skipped run onto 77, which SKIP_RETURN_CODE turns into +# "Skipped". multi_d is script-style-77 in its own right and is registered below. # ============================================================================= foreach(_gpu_test test_grouped_gemm_gpu_correctness @@ -184,12 +188,13 @@ foreach(_gpu_test test_mx_gemm_gpu_correctness) add_test( NAME dispatcher_${_gpu_test} - COMMAND ${Python3_EXECUTABLE} -m unittest discover -s ${CMAKE_CURRENT_SOURCE_DIR} -p ${_gpu_test}.py -v + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/run_unittest_77.py ${_gpu_test}.py WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) set_tests_properties(dispatcher_${_gpu_test} PROPERTIES LABELS "dispatcher;python;bridge;gpu" TIMEOUT 600 + SKIP_RETURN_CODE 77 ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" ) endforeach() @@ -220,13 +225,20 @@ set_tests_properties(dispatcher_test_multi_d_gpu_correctness PROPERTIES # nowhere -- it was reachable only from the Jenkins dispatcher lane, so a local # `ctest` never exercised it and neither did any pre-merge gate. # -# --budget 4 rather than the Jenkins lane's 16: each sampled config is its own -# hipcc build, and this is a correctness canary here, not the sweep. It already -# returns 77 when no GPU is present and --arch is not given (:281), so -# SKIP_RETURN_CODE keeps a CPU-only runner honest instead of green. +# --budget 16, one per (dtype x layout) stratum: the sampler splits the budget +# evenly across the 16 strata the standard variant enumerates, so anything below +# 16 leaves whole strata unsampled on any given run. At --budget 4 the run was +# not a small slice of the space but a slice of a quarter of it, and no single +# run ever compiled an fp8 or bf8 config. Each sampled config is its own hipcc +# build, hence the 1800s timeout. +# +# The runner returns 77 when no GPU is present and --arch is not given, so +# SKIP_RETURN_CODE keeps a CPU-only runner honest instead of green. That skip +# path and the sampler's stratum coverage are both pinned by +# test_search_space_sampling.py, which needs no GPU. add_test( NAME dispatcher_test_gemm_search_space - COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_gemm_search_space.py --budget 4 + COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_gemm_search_space.py --budget 16 WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) @@ -237,6 +249,26 @@ set_tests_properties(dispatcher_test_gemm_search_space PROPERTIES ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" ) +# Unit tests for the search-space runner's sampler and skip path. Pure CPU, no +# hipcc and no built kernels, so unlike the sweep above these run on every +# machine and gate every build. +# +# Registered separately because neither behaviour is observable from the sweep's +# own result: a sampler that draws from only a quarter of the strata still +# passes every config it does run, and a broken skip path shows up as a green +# GPU lane, not a red one. Both regressed silently once already. +add_test( + NAME dispatcher_test_search_space_sampling + COMMAND ${Python3_EXECUTABLE} -m pytest -q ${CMAKE_CURRENT_SOURCE_DIR}/test_search_space_sampling.py + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) + +set_tests_properties(dispatcher_test_search_space_sampling PROPERTIES + LABELS "dispatcher;python;gemm;validation" + TIMEOUT 60 + ENVIRONMENT "PYTHONPATH=${CMAKE_CURRENT_SOURCE_DIR}/../python:${CMAKE_CURRENT_SOURCE_DIR}/../codegen:${CMAKE_CURRENT_SOURCE_DIR}/../scripts" +) + # ============================================================================= # RowColQuant / TensorQuant bridge tests. # diff --git a/projects/composablekernel/dispatcher/tests/run_unittest_77.py b/projects/composablekernel/dispatcher/tests/run_unittest_77.py new file mode 100644 index 000000000000..41e0a7db7be7 --- /dev/null +++ b/projects/composablekernel/dispatcher/tests/run_unittest_77.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 + +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +""" +ctest driver for unittest.TestCase suites that skip themselves. + +``python -m unittest discover`` exits 0 when every test skipped, so ctest +records Passed for a run that executed nothing. On a CPU-only runner -- or a +gfx942 box for a gfx950-only suite -- that is a green tick backed by zero +assertions, which is exactly the signal SKIP_RETURN_CODE exists to remove. + +This runs the same discovery and maps the outcome onto the convention the +script-style tests here already use: + + 0 at least one test actually ran, and everything passed + 77 every collected test skipped -> ctest "Skipped" + 1 a test failed or errored, or discovery collected nothing at all + +Discovery collecting nothing is a failure rather than a skip on purpose: a +renamed file or a typo'd pattern otherwise reports Skipped forever and nobody +notices the suite left the build. + +Used instead of converting the suites to the script-style-77 form so the +skip-vs-run decision stays where it belongs -- in each test's own setUp. +""" + +import sys +import unittest +from pathlib import Path + +_SKIP = 77 + + +def main(argv) -> int: + if len(argv) != 2: + print(f"usage: {Path(argv[0]).name} ", file=sys.stderr) + return 2 + + pattern = argv[1] + start_dir = str(Path(__file__).resolve().parent) + + suite = unittest.defaultTestLoader.discover(start_dir, pattern=pattern) + result = unittest.TextTestRunner(verbosity=2).run(suite) + + if not result.wasSuccessful(): + return 1 + + if result.testsRun == 0: + print( + f"ERROR: pattern {pattern!r} collected no tests under {start_dir}. " + "The file was probably renamed or removed -- reporting failure " + "rather than Skipped so this does not go unnoticed.", + file=sys.stderr, + ) + return 1 + + ran = result.testsRun - len(result.skipped) + if ran == 0: + reasons = sorted({reason for _, reason in result.skipped}) + print( + f"SKIP: all {result.testsRun} test(s) in {pattern} skipped " + f"({'; '.join(reasons)})" + ) + return _SKIP + + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/projects/composablekernel/dispatcher/tests/test_abquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_abquant_gpu_correctness.py index 7e7ebfb2be0a..c85bddb07da2 100644 --- a/projects/composablekernel/dispatcher/tests/test_abquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_abquant_gpu_correctness.py @@ -312,7 +312,7 @@ def _run_one(label: str, config, M: int, N: int, K: int, dtype: str, # must be exact multiples of TileM, TileN and TileK. Shapes below are the # smallest multi-tile sizes that satisfy that for each pipeline. -def test_fp8_compv3(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": +def case_fp8_compv3(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": # compv3 tiles are 128x128x128; M=N=256 spans a 2x2 tile grid. # bquant_group_n=1 gives one B scale per column, the finest granularity. M, N, K, gK, bN = 256, 256, 512, 128, 1 @@ -322,7 +322,7 @@ def test_fp8_compv3(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": gfx_arch=gfx_arch, seed=42) -def test_bf8_compv3(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": +def case_bf8_compv3(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": M, N, K, gK, bN = 256, 256, 512, 128, 1 cfg = default_bf8_compv3_config(quant_group_k=gK, bquant_group_n=bN, gfx_arch=gfx_arch) @@ -330,7 +330,7 @@ def test_bf8_compv3(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": gfx_arch=gfx_arch, seed=43) -def test_fp8_eightwaves(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": +def case_fp8_eightwaves(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": if gfx_arch not in EIGHTWAVES_SUPPORTED_ARCHS: return SKIP, (f"fp8/eightwaves: not yet enabled on {gfx_arch} " f"(see EIGHTWAVES_SUPPORTED_ARCHS)") @@ -349,9 +349,9 @@ def test_fp8_eightwaves(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": # --------------------------------------------------------------------------- TESTS = [ - ("fp8/compv3", test_fp8_compv3), - ("bf8/compv3", test_bf8_compv3), - ("fp8/eightwaves", test_fp8_eightwaves), + ("fp8/compv3", case_fp8_compv3), + ("bf8/compv3", case_bf8_compv3), + ("fp8/eightwaves", case_fp8_eightwaves), ] diff --git a/projects/composablekernel/dispatcher/tests/test_aquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_aquant_gpu_correctness.py index 82baf1ae9f92..05557955e6cf 100644 --- a/projects/composablekernel/dispatcher/tests/test_aquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_aquant_gpu_correctness.py @@ -231,7 +231,7 @@ def _run_one(label: str, config, M: int, N: int, K: int, # Individual test cases # --------------------------------------------------------------------------- -def test_fp8(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": +def case_fp8(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": # K=768 = 3*TileK(256): num_loop=3 gives the no-hot-loop/TailNumber::Odd path, # which exercises the AQ scale prefetch more thoroughly than an even tail. M, N, K, gM, gK = 16, 64, 768, 1, 128 @@ -241,7 +241,7 @@ def test_fp8(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": out_dir, gfx_arch=gfx_arch) -def test_bf8(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": +def case_bf8(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": M, N, K, gM, gK = 16, 64, 768, 1, 128 cfg = default_bf8_config(quant_group_k=gK, quant_group_m=gM, gfx_arch=gfx_arch) A_raw, A_dec, B_raw, B_dec, AQ = _make_inputs(M, N, K, gM, gK, "bf8", gfx_arch, @@ -250,7 +250,7 @@ def test_bf8(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": out_dir, gfx_arch=gfx_arch) -def test_fp8_tiled(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": +def case_fp8_tiled(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": # M=64, N=256 spans 4x4 output tiles (tile_m=16, tile_n=64). The single-tile # cases above cannot distinguish a correct tile index from a constant one. M, N, K, gM, gK = 64, 256, 768, 1, 128 @@ -266,9 +266,9 @@ def test_fp8_tiled(out_dir: Path, gfx_arch: str) -> "tuple[str, str]": # --------------------------------------------------------------------------- TESTS = [ - ("fp8", test_fp8), - ("bf8", test_bf8), - ("fp8/tiled", test_fp8_tiled), + ("fp8", case_fp8), + ("bf8", case_bf8), + ("fp8/tiled", case_fp8_tiled), ] diff --git a/projects/composablekernel/dispatcher/tests/test_bquant_gpu_correctness.py b/projects/composablekernel/dispatcher/tests/test_bquant_gpu_correctness.py index d74d644bd58a..8a514fbd5b80 100644 --- a/projects/composablekernel/dispatcher/tests/test_bquant_gpu_correctness.py +++ b/projects/composablekernel/dispatcher/tests/test_bquant_gpu_correctness.py @@ -357,7 +357,7 @@ def _make_bf16_inputs(M, N, K, gK, gN, seed=42): _N_SWEEP = (128, 256, 512) -def test_c4_fp8(out_dir: Path, gfx_arch: str) -> tuple[str, str]: +def case_c4_fp8(out_dir: Path, gfx_arch: str) -> tuple[str, str]: # K=768 = 3*TileK(256): use num_loop=3 (TailNumber::Odd) for better coverage. # num_loop=2 works but exercises only the no-hot-loop/Even tail path; 3 gives # the no-hot-loop/Odd tail path and exercises the BQ scale prefetch more robustly. @@ -374,8 +374,8 @@ def test_c4_fp8(out_dir: Path, gfx_arch: str) -> tuple[str, str]: return PASS, f"C4/fp8: PASS for N in {_N_SWEEP}" -def test_c4_bf8(out_dir: Path, gfx_arch: str) -> tuple[str, str]: - # K=768 = 3*TileK(256): use num_loop=3 for the same reason as test_c4_fp8. +def case_c4_bf8(out_dir: Path, gfx_arch: str) -> tuple[str, str]: + # K=768 = 3*TileK(256): use num_loop=3 for the same reason as case_c4_fp8. M, K, gK, gN = 16, 768, 128, 1 cfg = default_bf8_config(quant_group_k=gK, quant_group_n=gN, gfx_arch=gfx_arch) for N in _N_SWEEP: @@ -389,7 +389,7 @@ def test_c4_bf8(out_dir: Path, gfx_arch: str) -> tuple[str, str]: return PASS, f"C4/bf8: PASS for N in {_N_SWEEP}" -def test_h3_mx_bf16bf16(out_dir: Path, gfx_arch: str) -> tuple[str, str]: +def case_h3_mx_bf16bf16(out_dir: Path, gfx_arch: str) -> tuple[str, str]: if gfx_arch not in MX_SUPPORTED_ARCHS: return SKIP, (f"H3/mx_bf16bf16: not supported on {gfx_arch} " f"(requires {'/'.join(MX_SUPPORTED_ARCHS)})") @@ -408,7 +408,7 @@ def test_h3_mx_bf16bf16(out_dir: Path, gfx_arch: str) -> tuple[str, str]: return PASS, f"H3/mx_bf16bf16: PASS for N in {_N_SWEEP}" -def test_h3_mx_bf16bf8(out_dir: Path, gfx_arch: str) -> tuple[str, str]: +def case_h3_mx_bf16bf8(out_dir: Path, gfx_arch: str) -> tuple[str, str]: if gfx_arch not in MX_SUPPORTED_ARCHS: return SKIP, (f"H3/mx_bf16bf8: not supported on {gfx_arch} " f"(requires {'/'.join(MX_SUPPORTED_ARCHS)})") @@ -430,7 +430,7 @@ def test_h3_mx_bf16bf8(out_dir: Path, gfx_arch: str) -> tuple[str, str]: return PASS, f"H3/mx_bf16bf8: PASS for N in {_N_SWEEP}" -def test_h3_mx_bf16fp4(out_dir: Path, gfx_arch: str) -> tuple[str, str]: +def case_h3_mx_bf16fp4(out_dir: Path, gfx_arch: str) -> tuple[str, str]: if gfx_arch not in MX_SUPPORTED_ARCHS: return SKIP, (f"H3/mx_bf16fp4: not supported on {gfx_arch} " f"(requires {'/'.join(MX_SUPPORTED_ARCHS)})") @@ -459,7 +459,7 @@ def test_h3_mx_bf16fp4(out_dir: Path, gfx_arch: str) -> tuple[str, str]: return PASS, f"H3/mx_bf16fp4: PASS for N in {_N_SWEEP}" -def test_c_i4(out_dir: Path, gfx_arch: str) -> tuple[str, str]: +def case_c_i4(out_dir: Path, gfx_arch: str) -> tuple[str, str]: """Round-6: fp8i4 / bf8i4 must be exact once BQ is encoded to the kernel's QDataType (fp8/bf8, 1 byte). The round-5 float32 BQ produced NaN. Swept over N to also exercise the per-N-tile de-permute. B is pk_int4 (2 per byte).""" @@ -515,12 +515,12 @@ def test_c_i4(out_dir: Path, gfx_arch: str) -> tuple[str, str]: # --------------------------------------------------------------------------- TESTS = [ - ("C4/fp8", test_c4_fp8), - ("C4/bf8", test_c4_bf8), - ("C/i4", test_c_i4), - ("H3/mx_bf16bf16", test_h3_mx_bf16bf16), - ("H3/mx_bf16bf8", test_h3_mx_bf16bf8), - ("H3/mx_bf16fp4", test_h3_mx_bf16fp4), + ("C4/fp8", case_c4_fp8), + ("C4/bf8", case_c4_bf8), + ("C/i4", case_c_i4), + ("H3/mx_bf16bf16", case_h3_mx_bf16bf16), + ("H3/mx_bf16bf8", case_h3_mx_bf16bf8), + ("H3/mx_bf16fp4", case_h3_mx_bf16fp4), ] diff --git a/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py b/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py index 0210e72c17e6..41693bf279f0 100644 --- a/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py +++ b/projects/composablekernel/dispatcher/tests/test_gemm_search_space.py @@ -121,18 +121,29 @@ def _daily_seed() -> int: def _sample(strata: dict, budget: int, seed: int) -> list: """Stratified budget-limited sample: equal share per non-empty stratum. - Strata are visited in sorted order so the sample is reproducible from the - seed alone. Each gets floor(budget/n), with the first ``budget % n`` strata - taking one extra; a stratum smaller than its share releases the difference - to a final top-up draw, so the full budget is used whenever the space allows. + Each stratum gets floor(budget/n), with ``budget % n`` strata taking one + extra; a stratum smaller than its share releases the difference to a final + top-up draw, so the full budget is used whenever the space allows. + + The strata are enumerated in sorted order and then *shuffled* before the + remainder is handed out, rather than being visited in sorted order. Sorted + order alone made the remainder deterministic in a way the seed could not + move: when ``budget < n`` every stratum's share is 0, the whole budget is + remainder, and it all lands on the lexicographically-first strata. At the + real 16 strata (bf16|bf8|fp16|fp8 x ccr|crr|rcr|rrr), --budget 4 therefore + picked bf16/{ccr,crr,rcr,rrr} for every seed, so the ctest registration was + permanently bf16-only and the daily seed bought nothing. Seeding the shuffle + from the same rng keeps the result reproducible from the seed alone. """ nonempty = {k: v for k, v in strata.items() if v} if not nonempty or budget <= 0: return [] rng = random.Random(seed) per, remainder = divmod(budget, len(nonempty)) + strata_order = sorted(nonempty.items()) + rng.shuffle(strata_order) selected, leftover = [], [] - for i, (_, configs) in enumerate(sorted(nonempty.items())): + for i, (_, configs) in enumerate(strata_order): # Clamp to the stratum size: budget < n makes `per` 0, and a stratum # smaller than its share would otherwise ask random.sample for more # items than exist. @@ -284,7 +295,13 @@ def _verify(variant: str, cfg, runner, result, ops: _Operands) -> float: def run(args) -> int: - arch = args.arch or detect_gpu_arch() + # fallback="" so a failed detection is falsy and the skip below can fire. + # detect_gpu_arch defaults to fallback="gfx942" (ctypes_utils.py), which + # never returns empty -- with the default, a CPU-only runner walked straight + # into hipcc instead of reporting skipped, which is the exact false signal + # the exit-77 convention exists to remove. Matches the aquant/abquant/bquant + # GPU tests, which all spell this out the same way. + arch = args.arch or detect_gpu_arch(fallback="") or None if not arch: print("SKIP: no GPU detected and --arch not given") return _SKIP diff --git a/projects/composablekernel/dispatcher/tests/test_search_space_sampling.py b/projects/composablekernel/dispatcher/tests/test_search_space_sampling.py new file mode 100644 index 000000000000..46eca129138d --- /dev/null +++ b/projects/composablekernel/dispatcher/tests/test_search_space_sampling.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 + +# Copyright (c) Advanced Micro Devices, Inc., or its affiliates. +# SPDX-License-Identifier: MIT +""" +Unit tests for test_gemm_search_space.py's sampler and skip path. + +Both behaviours regressed silently once before and neither is observable from +the runner's own output, so they are pinned here rather than left to the GPU +lane: these run on any box, with no GPU, hipcc or built kernels. +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + +_TESTS = Path(__file__).parent +sys.path.insert(0, str(_TESTS.parent / "python")) + + +def _load_runner(): + """Import the runner by path. + + It is a script whose name begins with ``test_``, so a plain ``import`` would + have pytest collect it as a second test module. + """ + spec = importlib.util.spec_from_file_location( + "_gemm_search_space_runner", _TESTS / "test_gemm_search_space.py" + ) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +runner = _load_runner() + +# The real 16 strata the standard variant enumerates. +_DTYPES = ["bf16", "bf8", "fp16", "fp8"] +_LAYOUTS = ["ccr", "crr", "rcr", "rrr"] + + +def _strata(per_stratum=10): + return { + f"{d}/{l}": [f"{d}/{l}/cfg{i}" for i in range(per_stratum)] + for d in _DTYPES + for l in _LAYOUTS + } + + +def _dtypes_of(selected): + return {s.split("/")[0] for s in selected} + + +def test_budget_below_stratum_count_varies_which_strata_are_picked(): + """budget < n_strata must let the seed move which *strata* are visited. + + The remainder used to be handed out in sorted stratum order, so with + ``budget=4`` over 16 strata every share was 0, the whole budget was + remainder, and it always landed on bf16/{ccr,crr,rcr,rrr}. The ctest + registration was therefore permanently bf16-only and the daily rotating + seed bought nothing. + + This asserts on the stratum keys, not on the selected configs: the old code + did vary the configs it drew from within those four fixed bf16 strata, so a + whole-sample comparison passes against the bug and pins nothing. + """ + strata = _strata() + visited = { + frozenset(s.rsplit("/", 1)[0] for s in runner._sample(strata, budget=4, seed=s)) + for s in range(20) + } + assert len(visited) > 1, "every seed drew from the identical four strata" + + +def test_budget_below_stratum_count_reaches_every_dtype_across_seeds(): + """Across seeds the union of picks must reach every dtype, not just bf16.""" + seen = set() + for seed in range(50): + seen |= _dtypes_of(runner._sample(_strata(), budget=4, seed=seed)) + assert seen == set(_DTYPES) + + +def test_sample_is_reproducible_from_seed(): + strata = _strata() + assert runner._sample(strata, budget=7, seed=42) == runner._sample( + strata, budget=7, seed=42 + ) + + +@pytest.mark.parametrize("budget", [1, 4, 15, 16, 17, 64, 160]) +def test_sample_never_exceeds_budget(budget): + selected = runner._sample(_strata(), budget=budget, seed=7) + assert len(selected) == budget + assert len(set(selected)) == len(selected), "sampled the same config twice" + + +def test_sample_handles_strata_smaller_than_their_share(): + """A short stratum releases its slack instead of over-drawing. + + ``random.sample`` raises if asked for more items than exist, so the clamp + and the top-up draw have to agree. + """ + strata = {"bf16/rcr": ["a", "b"], "fp16/rcr": [f"c{i}" for i in range(50)]} + selected = runner._sample(strata, budget=20, seed=3) + assert len(selected) == 20 + assert len(set(selected)) == 20 + + +def test_sample_caps_at_total_when_budget_exceeds_space(): + strata = {"bf16/rcr": ["a", "b"], "fp16/rcr": ["c"]} + assert sorted(runner._sample(strata, budget=99, seed=1)) == ["a", "b", "c"] + + +def test_sample_empty_inputs(): + assert runner._sample({}, budget=10, seed=1) == [] + assert runner._sample({"bf16/rcr": []}, budget=10, seed=1) == [] + assert runner._sample(_strata(), budget=0, seed=1) == [] + + +def test_no_gpu_exits_77(monkeypatch, capsys): + """A CPU-only runner must report skipped, not walk into hipcc. + + ``detect_gpu_arch`` defaults to ``fallback="gfx942"`` and so never returns + falsy; calling it bare made this skip path dead code and a GPU-less box + compiled kernels for an arch it does not have. + + The stub returns the fallback it is handed rather than a fixed "", which is + what the real function does when rocminfo finds nothing. That is what makes + this test discriminating: the runner only skips if it asks for ``fallback=""``. + Stubbing a bare "" would pass against the old bare ``detect_gpu_arch()`` call + too, and pin nothing. + """ + monkeypatch.setattr(runner, "detect_gpu_arch", lambda fallback="gfx942": fallback) + monkeypatch.setattr(sys, "argv", ["test_gemm_search_space.py", "--budget", "1"]) + assert runner.main() == runner._SKIP + assert "SKIP" in capsys.readouterr().out + + +def test_unsupported_arch_exits_77(capsys): + """An unsupported GPU skips too, rather than failing every hipcc step.""" + monkeypatch = pytest.MonkeyPatch() + monkeypatch.setattr(sys, "argv", ["x", "--arch", "gfx90a", "--budget", "1"]) + try: + assert runner.main() == runner._SKIP + finally: + monkeypatch.undo() + assert "SKIP" in capsys.readouterr().out + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) From e615e9dd55413be41a16a63ce17f1d9a37805084 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Tue, 8 Sep 2026 16:41:49 -0400 Subject: [PATCH 49/51] fix(ck-tile): expand DISPATCHER_PERF gfx950 to full operator set The gfx950 perf lane previously ran only gemm_aquant, gemm_bquant, gemm_abquant, mx_gemm, and gemm_streamk. Every other operator -- gemm_universal, batched_gemm, batched_contraction, grouped_gemm, gemm_multi_d, gemm_multi_abd, grouped_gemm_rowcolquant, grouped_gemm_tensorquant -- was CMake-legal on gfx950 (all list gfx950 in DESIRED_TARGETS) but absent from the gfx950 command because _dispatcherPerfCmd used a hard arch branch that built a completely different operator set on each side. Collapse the branch into a shared base (all arch-agnostic ops) with a gfx950-only appendix (gemm_aquant/bquant/abquant/mx_gemm). GPU_TARGETS is now passed through from the caller rather than hardcoded. Remaining gfx950-only set and why: gemm_aquant/bquant/abquant SUPPORTED_ARCHS = ("gfx950",) in their Python layers; gfx942 has fp8 FNUZ and is in DESIRED_TARGETS, but the guards have not been lifted yet. mx_gemm CMakeLists filters with target MATCHES "^gfx950", so benchmark_mx_gemm_all is never created for gfx942; also has no fp16 path, hence quantDatatype. Deferred (next PR, same list as the correctness lane): gemm_preshuffle no dispatcher bridge on develop contraction_multi_abd no Python benchmark driver yet Co-Authored-By: Claude Opus 5 --- .../composablekernel/groovy/vars/ck.groovy | 117 ++++++++++-------- 1 file changed, 62 insertions(+), 55 deletions(-) diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 29cfbfb32769..1ae445ee00bb 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1409,42 +1409,28 @@ String _dispatcherPerfCmd(String compiler, String gpuTarget, String samplingTier String gemmDatatype, String gemmLayout, String multiLayout, String quantDatatype, String quantLayout, String problemSizes, String problemConfigs, String perOpInstances) { - if (gpuTarget == "gfx950") { - return """ - cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ - -D BUILD_CK_TILE_ENGINE="ON" \ - -D CMAKE_CXX_COMPILER="${compiler}" \ - -D CMAKE_BUILD_TYPE=Release \ - -D GPU_TARGETS="gfx950" \ - -D GEMM_AQUANT_DATATYPE="${quantDatatype}" \ - -D GEMM_AQUANT_LAYOUT="${quantLayout}" \ - -D GEMM_AQUANT_MAX_INSTANCES=${perOpInstances} \ - -D GEMM_BQUANT_DATATYPE="${quantDatatype}" \ - -D GEMM_BQUANT_LAYOUT="${quantLayout}" \ - -D GEMM_BQUANT_MAX_INSTANCES=${perOpInstances} \ - -D GEMM_ABQUANT_DATATYPE="${quantDatatype}" \ - -D GEMM_ABQUANT_LAYOUT="${quantLayout}" \ - -D GEMM_ABQUANT_MAX_INSTANCES=${perOpInstances} \ - -D MX_GEMM_DATATYPE="${quantDatatype}" \ - -D MX_GEMM_LAYOUT="${quantLayout}" \ - -D MX_GEMM_MAX_INSTANCES=${perOpInstances} \ - -D GEMM_STREAMK_DATATYPE="${gemmDatatype}" \ - -D GEMM_STREAMK_LAYOUT="${gemmLayout}" \ - -D GEMM_STREAMK_MAX_INSTANCES=${perOpInstances} \ - -D TILE_ENGINE_SAMPLING_TIER=${samplingTier} .. && \ - ninja -j\$(nproc) benchmark_gemm_aquant_all benchmark_gemm_bquant_all benchmark_gemm_abquant_all benchmark_mx_gemm_all benchmark_gemm_streamk_all && \ - python3 ../tile_engine/ops/gemm/block_scale_gemm/gemm_aquant/gemm_aquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_aquant_results.json && \ - python3 ../tile_engine/ops/gemm/block_scale_gemm/gemm_bquant/gemm_bquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_bquant_results.json && \ - python3 ../tile_engine/ops/gemm/block_scale_gemm/gemm_abquant/gemm_abquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_abquant_results.json && \ - python3 ../tile_engine/ops/gemm/mx_gemm/mx_gemm_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json mx_gemm_results.json && \ - python3 ../tile_engine/ops/gemm_streamk/gemm_streamk_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_streamk_results.json""" - } - return """ + // All operators whose CMakeLists list both gfx942 and gfx950 in DESIRED_TARGETS + // run on both arches. Two sets remain gfx950-only: + // + // gemm_aquant / gemm_bquant / gemm_abquant: SUPPORTED_ARCHS = ("gfx950",) in + // their Python layers. The C++ side would compile on gfx942 (standard fp8 + // MFMA), but the Python guards have not been lifted yet. Widening is its own + // change; the gate is conservative and likely to lift soon. + // + // mx_gemm: CMakeLists filters with `target MATCHES "^gfx950"`, so + // benchmark_mx_gemm_all is never created for gfx942 and ninja would fail on + // an unknown target. Also has no fp16 path (MX_GEMM_DATATYPE defaults to + // fp4;fp8), hence quantDatatype below. + // + // gemm_preshuffle and contraction_multi_abd are deferred: preshuffle has no + // dispatcher bridge on develop; contraction_multi_abd has no Python benchmark + // driver. Both get lanes in the next PR once those prerequisites land. + def cmd = """ cmake -G Ninja -D CMAKE_PREFIX_PATH=/opt/rocm \ -D BUILD_CK_TILE_ENGINE="ON" \ -D CMAKE_CXX_COMPILER="${compiler}" \ -D CMAKE_BUILD_TYPE=Release \ - -D GPU_TARGETS="gfx942" \ + -D GPU_TARGETS="${gpuTarget}" \ -D GEMM_UNIVERSAL_DATATYPE="${gemmDatatype}" \ -D GEMM_UNIVERSAL_LAYOUT="${gemmLayout}" \ -D GEMM_UNIVERSAL_MAX_INSTANCES=${perOpInstances} \ @@ -1471,9 +1457,29 @@ String _dispatcherPerfCmd(String compiler, String gpuTarget, String samplingTier -D GROUPED_GEMM_ROWCOLQUANT_MAX_INSTANCES=${perOpInstances} \ -D GROUPED_GEMM_TENSORQUANT_DATATYPE="${quantDatatype}" \ -D GROUPED_GEMM_TENSORQUANT_LAYOUT="${quantLayout}" \ - -D GROUPED_GEMM_TENSORQUANT_MAX_INSTANCES=${perOpInstances} \ + -D GROUPED_GEMM_TENSORQUANT_MAX_INSTANCES=${perOpInstances}""" + if (gpuTarget == "gfx950") { + cmd += """ \ + -D GEMM_AQUANT_DATATYPE="${quantDatatype}" \ + -D GEMM_AQUANT_LAYOUT="${quantLayout}" \ + -D GEMM_AQUANT_MAX_INSTANCES=${perOpInstances} \ + -D GEMM_BQUANT_DATATYPE="${quantDatatype}" \ + -D GEMM_BQUANT_LAYOUT="${quantLayout}" \ + -D GEMM_BQUANT_MAX_INSTANCES=${perOpInstances} \ + -D GEMM_ABQUANT_DATATYPE="${quantDatatype}" \ + -D GEMM_ABQUANT_LAYOUT="${quantLayout}" \ + -D GEMM_ABQUANT_MAX_INSTANCES=${perOpInstances} \ + -D MX_GEMM_DATATYPE="${quantDatatype}" \ + -D MX_GEMM_LAYOUT="${quantLayout}" \ + -D MX_GEMM_MAX_INSTANCES=${perOpInstances}""" + } + cmd += """ \ -D TILE_ENGINE_SAMPLING_TIER=${samplingTier} .. && \ - ninja -j\$(nproc) benchmark_gemm_universal_all benchmark_batched_gemm_all benchmark_batched_contraction_all benchmark_gemm_streamk_all benchmark_grouped_gemm_all benchmark_gemm_multi_d_all benchmark_gemm_multi_abd_all benchmark_grouped_gemm_rowcolquant_all benchmark_grouped_gemm_tensorquant_all && \ + ninja -j\$(nproc) benchmark_gemm_universal_all benchmark_batched_gemm_all benchmark_batched_contraction_all benchmark_gemm_streamk_all benchmark_grouped_gemm_all benchmark_gemm_multi_d_all benchmark_gemm_multi_abd_all benchmark_grouped_gemm_rowcolquant_all benchmark_grouped_gemm_tensorquant_all""" + if (gpuTarget == "gfx950") { + cmd += " benchmark_gemm_aquant_all benchmark_gemm_bquant_all benchmark_gemm_abquant_all benchmark_mx_gemm_all" + } + cmd += """ && \ python3 ../tile_engine/ops/gemm/gemm_universal/gemm_universal_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_universal_results.json && \ python3 ../tile_engine/ops/gemm/batched_gemm/batched_gemm_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json batched_gemm_results.json && \ python3 ../tile_engine/ops/gemm/batched_contraction/batched_contraction_benchmark.py . --problem-configs ${problemConfigs} --warmup 5 --repeat 5 --verbose --json batched_contraction_results.json && \ @@ -1483,6 +1489,14 @@ String _dispatcherPerfCmd(String compiler, String gpuTarget, String samplingTier python3 ../tile_engine/ops/gemm/gemm_multi_abd/gemm_multi_abd_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_multi_abd_results.json && \ python3 ../tile_engine/ops/gemm/grouped_gemm_quant/grouped_gemm_rowcolquant/grouped_gemm_rowcolquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json grouped_gemm_rowcolquant_results.json && \ python3 ../tile_engine/ops/gemm/grouped_gemm_quant/grouped_gemm_tensorquant/grouped_gemm_tensorquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json grouped_gemm_tensorquant_results.json""" + if (gpuTarget == "gfx950") { + cmd += """ && \ + python3 ../tile_engine/ops/gemm/block_scale_gemm/gemm_aquant/gemm_aquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_aquant_results.json && \ + python3 ../tile_engine/ops/gemm/block_scale_gemm/gemm_bquant/gemm_bquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_bquant_results.json && \ + python3 ../tile_engine/ops/gemm/block_scale_gemm/gemm_abquant/gemm_abquant_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json gemm_abquant_results.json && \ + python3 ../tile_engine/ops/gemm/mx_gemm/mx_gemm_benchmark.py . --problem-sizes ${problemSizes} --warmup 5 --repeat 5 --verbose --json mx_gemm_results.json""" + } + return cmd } // Benchmark ahead-of-time tile_engine instances at smoke coverage: one dtype, @@ -1521,30 +1535,23 @@ def runDispatcherPerfTests(String compiler, String gpuTarget = "gfx942") { // with integer truncation, so a multi-combo op can floor a combo to zero // instances and build nothing. def perOpInstances = "64" - // gemm_aquant and gemm_bquant run on gfx950 because their tests and utils - // modules self-gate to it -- SUPPORTED_ARCHS = ("gfx950",) and - // _DEFAULT_GFX_ARCH = "gfx950" -- not because gfx942 lacks fp8. gfx942 has - // fp8 (FNUZ) and is in DESIRED_TARGETS; widening these ops to it means - // relaxing those guards first. + // Operator distribution across arches -- see _dispatcherPerfCmd for the + // per-op rationale. Summary: // - // mx_gemm is on gfx950 for a harder reason: its CMakeLists filters targets - // with a literal regex, target MATCHES "^gfx950", so on any other arch the - // benchmark_mx_gemm_all target is never created and ninja fails on an - // unknown target rather than skipping. It also has no fp16 path at all - // (MX_GEMM_DATATYPE defaults to fp4;fp8), hence quantDatatype below. + // Both arches: gemm_universal, batched_gemm, batched_contraction, + // gemm_streamk, grouped_gemm, gemm_multi_d, gemm_multi_abd, + // grouped_gemm_rowcolquant, grouped_gemm_tensorquant. + // All have gfx950 in their DESIRED_TARGETS and no Python-level arch gate. // - // gemm_streamk is built on both arches. It used to be gfx942-only here - // because its DESIRED_TARGETS was "gfx90a;gfx942" with a TODO for gfx950; - // that list now includes gfx950, but the commit that widened it flagged the - // CMake change as the one part it could not exercise locally and left it for - // CI to confirm. Building it on gfx950 here is that confirmation, and it also - // matches runDispatcherCorrectnessTests, which already runs stream_k on both. - // If the gfx950 AOT build turns out to be broken, the DESIRED_TARGETS line is - // revertible on its own without touching this lane. + // gfx950 only: gemm_aquant, gemm_bquant, gemm_abquant, mx_gemm. + // gemm_aquant/bquant/abquant: SUPPORTED_ARCHS = ("gfx950",) in the Python + // layer. gfx942 has fp8 (FNUZ), but those guards haven't been lifted yet. + // mx_gemm: CMakeLists filters with `target MATCHES "^gfx950"`, so the + // benchmark_mx_gemm_all target is never created for gfx942; also has no + // fp16 path, hence quantDatatype below. // - // Everything else -- gemm_universal, batched_gemm, batched_contraction, - // grouped_gemm, gemm_multi_d, gemm_multi_abd and the two grouped quant ops -- - // is CMake-legal on both, and sits on gfx942 to keep the two stages balanced. + // Deferred (next PR): gemm_preshuffle (no bridge on develop), and + // contraction_multi_abd (no Python benchmark driver yet). def quantDatatype = "fp8" def quantLayout = "rcr" // Stream-K below is tile_engine/ops/gemm_streamk/ (top level), not From b413908d67cb28980f75a2fccb7524a0ea18f0e4 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Tue, 8 Sep 2026 19:13:37 -0400 Subject: [PATCH 50/51] fix(ck): inline runDownstreamTestsStages into Jenkinsfile Jenkins ignores the dynamic library("ck@branch") load when ck is already registered as a global shared library, causing the call to fail with 'No such DSL method'. Inline the parallel block directly in the Jenkinsfile and drop the groovy wrapper. Co-Authored-By: Claude --- projects/composablekernel/Jenkinsfile | 43 ++++++++++++++++- .../composablekernel/groovy/vars/ck.groovy | 47 +------------------ 2 files changed, 43 insertions(+), 47 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index 57e24a1d2554..56aeaff843da 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -387,7 +387,48 @@ pipeline { steps { script { loadCk() - ck.runDownstreamTestsStages(this.&rocmnode, params) + parallel([ + "Run Pytorch Tests on gfx942": { + if (params.RUN_PYTORCH_TESTS.toBoolean()) { + ck.runOnHealthyNode(rocmnode("gfx942")) { + ck.run_downstream_tests(image: "${env.CK_PYTORCH_IMAGE}", timeoutHours: 2, execute_cmds: ck.getPytorchTestsCmds()) + cleanWs() + } + } + }, + "Run AITER Tests on gfx942": { + if (params.RUN_AITER_TESTS.toBoolean()) { + ck.runOnHealthyNode(rocmnode("gfx942")) { + ck.run_downstream_tests(image: "${env.CK_AITER_IMAGE}", timeoutHours: 5, execute_cmds: ck.getAiterTestsCmds()) + cleanWs() + } + } + }, + "Run AITER Tests on gfx950": { + if (params.RUN_AITER_TESTS.toBoolean()) { + ck.runOnHealthyNode(rocmnode("gfx950")) { + ck.run_downstream_tests(image: "${env.CK_AITER_IMAGE}", timeoutHours: 5, execute_cmds: ck.getAiterTestsCmds()) + cleanWs() + } + } + }, + "Run FA Tests on gfx942": { + if (params.RUN_FA_TESTS.toBoolean()) { + ck.runOnHealthyNode(rocmnode("gfx942")) { + ck.run_downstream_tests(image: "${env.CK_FA_IMAGE}", timeoutHours: 5, execute_cmds: ck.getFaTestsCmds()) + cleanWs() + } + } + }, + "Run FA Tests on gfx950": { + if (params.RUN_FA_TESTS.toBoolean()) { + ck.runOnHealthyNode(rocmnode("gfx950")) { + ck.run_downstream_tests(image: "${env.CK_FA_IMAGE}", timeoutHours: 5, execute_cmds: ck.getFaTestsCmds()) + cleanWs() + } + } + } + ]) } } } diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index bcff8cd28dac..858804751f6a 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1864,7 +1864,7 @@ def runDispatcherCorrectnessTests(String arch, String compiler) { // Jenkinsfile (`def rocmnode(name)`), and a vars/ script does not inherit // Jenkinsfile-local methods, so calling it here throws MissingMethodException // before runOnHealthyNode is ever reached. Same convention as -// runDownstreamTestsStages / runFMHATestsStages / runBuildCKStages; the caller +// runFMHATestsStages / runBuildCKStages; the caller // passes `this.&rocmnode`. def runDispatcherTests(def rocmnode, boolean runCorrectness, boolean runPerf, String compiler) { def branches = [:] @@ -1957,51 +1957,6 @@ def runBuildInstancesOnly(String compiler) { ) } -def runDownstreamTestsStages(def rocmnode, def params) { - parallel([ - "Run Pytorch Tests on gfx942": { - if (params.RUN_PYTORCH_TESTS.toBoolean()) { - runOnHealthyNode(rocmnode("gfx942")) { - run_downstream_tests(image: "${env.CK_PYTORCH_IMAGE}", timeoutHours: 2, execute_cmds: getPytorchTestsCmds()) - cleanWs() - } - } - }, - "Run AITER Tests on gfx942": { - if (params.RUN_AITER_TESTS.toBoolean()) { - runOnHealthyNode(rocmnode("gfx942")) { - run_downstream_tests(image: "${env.CK_AITER_IMAGE}", timeoutHours: 5, execute_cmds: getAiterTestsCmds()) - cleanWs() - } - } - }, - "Run AITER Tests on gfx950": { - if (params.RUN_AITER_TESTS.toBoolean()) { - runOnHealthyNode(rocmnode("gfx950")) { - run_downstream_tests(image: "${env.CK_AITER_IMAGE}", timeoutHours: 5, execute_cmds: getAiterTestsCmds()) - cleanWs() - } - } - }, - "Run FA Tests on gfx942": { - if (params.RUN_FA_TESTS.toBoolean()) { - runOnHealthyNode(rocmnode("gfx942")) { - run_downstream_tests(image: "${env.CK_FA_IMAGE}", timeoutHours: 5, execute_cmds: getFaTestsCmds()) - cleanWs() - } - } - }, - "Run FA Tests on gfx950": { - if (params.RUN_FA_TESTS.toBoolean()) { - runOnHealthyNode(rocmnode("gfx950")) { - run_downstream_tests(image: "${env.CK_FA_IMAGE}", timeoutHours: 5, execute_cmds: getFaTestsCmds()) - cleanWs() - } - } - } - ]) -} - def runFMHATestsStages(def rocmnode, def params) { parallel([ "Run CK_TILE_FMHA Tests on gfx90a": { From 51ed93b00e6e57d531005b6e0f48931b088087f8 Mon Sep 17 00:00:00 2001 From: ThruptiRajLakshmanaGowda Date: Tue, 8 Sep 2026 19:53:20 -0400 Subject: [PATCH 51/51] fix(ck): inline runFMHATestsStages and runBuildCKStages into Jenkinsfile Same fix as runDownstreamTestsStages: Jenkins ignores the dynamic library load when ck is already registered globally, so any function that exists only on this branch is invisible at call time. Co-Authored-By: Claude --- projects/composablekernel/Jenkinsfile | 114 ++++++++++++++++- .../composablekernel/groovy/vars/ck.groovy | 119 +----------------- 2 files changed, 113 insertions(+), 120 deletions(-) diff --git a/projects/composablekernel/Jenkinsfile b/projects/composablekernel/Jenkinsfile index 56aeaff843da..b52fe8142e38 100644 --- a/projects/composablekernel/Jenkinsfile +++ b/projects/composablekernel/Jenkinsfile @@ -526,7 +526,44 @@ pipeline { steps { script { loadCk() - ck.runFMHATestsStages(this.&rocmnode, params) + parallel([ + "Run CK_TILE_FMHA Tests on gfx90a": { + if (params.RUN_CK_TILE_FMHA_TESTS.toBoolean()) { + ck.runOnHealthyNode(rocmnode("gfx90a")) { + deleteDir() + ck.buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: ck.build_and_run_fmha("gfx90a")) + cleanWs() + } + } + }, + "Run CK_TILE_FMHA Tests on gfx942": { + if (params.RUN_CK_TILE_FMHA_TESTS.toBoolean()) { + ck.runOnHealthyNode(rocmnode("gfx942")) { + deleteDir() + ck.buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: ck.build_and_run_fmha("gfx942")) + cleanWs() + } + } + }, + "Run CK_TILE_FMHA Tests on gfx950": { + if (params.RUN_CK_TILE_FMHA_TESTS.toBoolean()) { + ck.runOnHealthyNode(rocmnode("gfx950")) { + deleteDir() + ck.buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: ck.build_and_run_fmha("gfx950")) + cleanWs() + } + } + }, + "Run CK_TILE_FMHA Tests on gfx1201": { + if (params.RUN_CK_TILE_FMHA_TESTS.toBoolean()) { + ck.runOnHealthyNode(rocmnode("gfx1201")) { + deleteDir() + ck.buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: ck.build_and_run_fmha("gfx1201")) + cleanWs() + } + } + } + ]) } } } @@ -559,7 +596,80 @@ pipeline { steps { script { loadCk() - ck.runBuildCKStages(this.&rocmnode, params) + parallel([ + "Build CK and run Tests on gfx942": { + if ((params.BUILD_GFX942.toBoolean() || params.RUN_FULL_QA.toBoolean()) && !params.BUILD_INSTANCES_ONLY.toBoolean()) { + ck.runOnHealthyNode(rocmnode("gfx942")) { + deleteDir() + ck.runBuildCKAndTests("gfx942") + cleanWs() + } + } + }, + "Build CK and run Tests on gfx950": { + if (params.BUILD_GFX950.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean()) { + ck.runOnHealthyNode(rocmnode("gfx950")) { + deleteDir() + ck.runBuildCKAndTests("gfx950") + cleanWs() + } + } + }, + "Build CK and run Tests on gfx90a": { + if (params.BUILD_GFX90A.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean()) { + ck.runOnHealthyNode(rocmnode("gfx90a")) { + deleteDir() + ck.runBuildCKAndTests("gfx90a") + cleanWs() + } + } + }, + "Build CK instances for all supported targets": { + if (params.BUILD_INSTANCES_ONLY.toBoolean() && !params.RUN_FULL_QA.toBoolean()) { + ck.runOnHealthyNode(rocmnode("gfx942")) { + deleteDir() + ck.runBuildInstancesOnly(params.BUILD_COMPILER) + cleanWs() + } + } + }, + "Build CK and run Tests on gfx1030": { + if (params.BUILD_GFX103.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean()) { + ck.runOnHealthyNode(rocmnode("gfx1030")) { + deleteDir() + ck.runBuildCKAndTests("gfx10-3-generic") + cleanWs() + } + } + }, + "Build CK and run Tests on gfx11": { + if (params.BUILD_GFX11.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean()) { + ck.runOnHealthyNode('miopen && (gfx1101 || gfx1100)') { + deleteDir() + ck.runBuildCKAndTests("gfx11-generic") + cleanWs() + } + } + }, + "Build CK and run Tests on gfx1201": { + if (params.BUILD_GFX12.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean()) { + ck.runOnHealthyNode(rocmnode("gfx1201")) { + deleteDir() + ck.runBuildCKAndTests("gfx12-generic") + cleanWs() + } + } + }, + "Build CK for gfx1250": { + if (params.BUILD_GFX1250.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean()) { + ck.runOnHealthyNode(rocmnode("gfx90a")) { + deleteDir() + ck.runBuildCKAndTests("gfx1250") + cleanWs() + } + } + } + ]) } } post { diff --git a/projects/composablekernel/groovy/vars/ck.groovy b/projects/composablekernel/groovy/vars/ck.groovy index 858804751f6a..998445a8c607 100644 --- a/projects/composablekernel/groovy/vars/ck.groovy +++ b/projects/composablekernel/groovy/vars/ck.groovy @@ -1864,7 +1864,7 @@ def runDispatcherCorrectnessTests(String arch, String compiler) { // Jenkinsfile (`def rocmnode(name)`), and a vars/ script does not inherit // Jenkinsfile-local methods, so calling it here throws MissingMethodException // before runOnHealthyNode is ever reached. Same convention as -// runFMHATestsStages / runBuildCKStages; the caller +// the caller // passes `this.&rocmnode`. def runDispatcherTests(def rocmnode, boolean runCorrectness, boolean runPerf, String compiler) { def branches = [:] @@ -1957,120 +1957,3 @@ def runBuildInstancesOnly(String compiler) { ) } -def runFMHATestsStages(def rocmnode, def params) { - parallel([ - "Run CK_TILE_FMHA Tests on gfx90a": { - if (params.RUN_CK_TILE_FMHA_TESTS.toBoolean()) { - runOnHealthyNode(rocmnode("gfx90a")) { - deleteDir() - buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: build_and_run_fmha("gfx90a")) - cleanWs() - } - } - }, - "Run CK_TILE_FMHA Tests on gfx942": { - if (params.RUN_CK_TILE_FMHA_TESTS.toBoolean()) { - runOnHealthyNode(rocmnode("gfx942")) { - deleteDir() - buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: build_and_run_fmha("gfx942")) - cleanWs() - } - } - }, - "Run CK_TILE_FMHA Tests on gfx950": { - if (params.RUN_CK_TILE_FMHA_TESTS.toBoolean()) { - runOnHealthyNode(rocmnode("gfx950")) { - deleteDir() - buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: build_and_run_fmha("gfx950")) - cleanWs() - } - } - }, - "Run CK_TILE_FMHA Tests on gfx1201": { - if (params.RUN_CK_TILE_FMHA_TESTS.toBoolean()) { - runOnHealthyNode(rocmnode("gfx1201")) { - deleteDir() - buildAndTest(setup_args: "NO_CK_BUILD", build_type: 'Release', execute_cmd: build_and_run_fmha("gfx1201")) - cleanWs() - } - } - } - ]) -} - -def runBuildCKStages(def rocmnode, def params) { - parallel([ - "Build CK and run Tests on gfx942": { - if ((params.BUILD_GFX942.toBoolean() || params.RUN_FULL_QA.toBoolean()) && !params.BUILD_INSTANCES_ONLY.toBoolean()) { - runOnHealthyNode(rocmnode("gfx942")) { - deleteDir() - runBuildCKAndTests("gfx942") - cleanWs() - } - } - }, - "Build CK and run Tests on gfx950": { - if (params.BUILD_GFX950.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean()) { - runOnHealthyNode(rocmnode("gfx950")) { - deleteDir() - runBuildCKAndTests("gfx950") - cleanWs() - } - } - }, - "Build CK and run Tests on gfx90a": { - if (params.BUILD_GFX90A.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean()) { - runOnHealthyNode(rocmnode("gfx90a")) { - deleteDir() - runBuildCKAndTests("gfx90a") - cleanWs() - } - } - }, - "Build CK instances for all supported targets": { - if (params.BUILD_INSTANCES_ONLY.toBoolean() && !params.RUN_FULL_QA.toBoolean()) { - runOnHealthyNode(rocmnode("gfx942")) { - deleteDir() - runBuildInstancesOnly(params.BUILD_COMPILER) - cleanWs() - } - } - }, - "Build CK and run Tests on gfx1030": { - if (params.BUILD_GFX103.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean()) { - runOnHealthyNode(rocmnode("gfx1030")) { - deleteDir() - runBuildCKAndTests("gfx10-3-generic") - cleanWs() - } - } - }, - "Build CK and run Tests on gfx11": { - if (params.BUILD_GFX11.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean()) { - runOnHealthyNode('miopen && (gfx1101 || gfx1100)') { - deleteDir() - runBuildCKAndTests("gfx11-generic") - cleanWs() - } - } - }, - "Build CK and run Tests on gfx1201": { - if (params.BUILD_GFX12.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean()) { - runOnHealthyNode(rocmnode("gfx1201")) { - deleteDir() - runBuildCKAndTests("gfx12-generic") - cleanWs() - } - } - }, - "Build CK for gfx1250": { - if (params.BUILD_GFX1250.toBoolean() && !params.RUN_FULL_QA.toBoolean() && !params.BUILD_INSTANCES_ONLY.toBoolean()) { - runOnHealthyNode(rocmnode("gfx90a")) { - deleteDir() - runBuildCKAndTests("gfx1250") - cleanWs() - } - } - } - ]) -}