From 79ac6945a81d4e618b6083fe981e37e73c7aabfb Mon Sep 17 00:00:00 2001 From: Parthiv Aneesh Date: Mon, 19 Oct 2026 22:23:26 +0000 Subject: [PATCH 1/2] test(rocke): fix gfx942 dense CFVST/KDB/VPF benchmark coverage --- .../gfx942/attention/prefill/bench_rocke_5way_specs.py | 6 ++---- .../rocke/library/kernels/gfx942/attention_dense.py | 9 --------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill/bench_rocke_5way_specs.py b/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill/bench_rocke_5way_specs.py index 4b08b3744914..bd4d832b387c 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill/bench_rocke_5way_specs.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill/bench_rocke_5way_specs.py @@ -111,19 +111,17 @@ def locate_rocke(explicit: str | None) -> Path: starts = [] if explicit: starts.append(Path(explicit).expanduser()) - if os.environ.get("ROCKE_ROOT"): - starts.append(Path(os.environ["ROCKE_ROOT"]).expanduser()) starts.extend((Path.cwd(), Path(__file__).resolve().parent)) for start in starts: for path in (start, *start.parents): for candidate in (path, path / "dnn-providers/hip-kernel-provider/rocke"): if (candidate / "library/kernels/gfx942/attention_dense.py").is_file(): return candidate.resolve() - raise SystemExit("ROCKE root not found: set --rocke-root or ROCKE_ROOT") + raise SystemExit("ROCKE root not found: pass --rocke-root") def add_module_paths(rocke: Path) -> None: - for p in (rocke / "library", rocke / "platform/Python", rocke): + for p in (rocke / "library", rocke / "platform/python", rocke): if str(p) not in sys.path: sys.path.insert(0, str(p)) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx942/attention_dense.py b/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx942/attention_dense.py index daa85a718324..cb6a860ea1a8 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx942/attention_dense.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx942/attention_dense.py @@ -890,15 +890,6 @@ def _use_vpf(spec: "Gfx942AttentionDenseSpec") -> bool: return spec.v_reg_prefetch -def _use_kdb_vpf(spec: "Gfx942AttentionDenseSpec") -> bool: - """Combined variant predicate kept for existing external callers. - - Neither knob implies the other. The kernel's allocation, scheduling, and - naming use ``_use_kdb`` and ``_use_vpf`` independently. - """ - return _use_kdb(spec) and _use_vpf(spec) - - def _v_swizzle_width(block_n: int) -> int: """V^T row width (elements) the XOR bank-conflict swizzle needs: the smallest pow2 >= max(64, block_n). SINGLE source of truth -- the derived pad (:func:`_v_row_pad`), From 0f24ca81aaa1181cce4f52ea802f02024a278813 Mon Sep 17 00:00:00 2001 From: Parthiv Aneesh Date: Mon, 19 Oct 2026 22:23:31 +0000 Subject: [PATCH 2/2] docs(rocke): update gfx942 dense case study and benchmark reproduction --- .../gfx942/attention/prefill/README.md | 23 ++++---- ...42_dense_cfvst_kdb_vprefetch_case_study.md | 57 ++++++++++--------- 2 files changed, 41 insertions(+), 39 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx942/attention/prefill/README.md b/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx942/attention/prefill/README.md index a1f622516508..8d04693a2db1 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx942/attention/prefill/README.md +++ b/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx942/attention/prefill/README.md @@ -68,7 +68,7 @@ faster than the fixed overhead. Two things worth knowing about how to read the t earlier comparison, run before the pad was adopted, measured D64 at −17 % (fp16, short) to +29 %. Re-running with the pad in place is what removed the regression: an explicit ON/OFF A/B in the same harness puts it at **1.88–2.13× (fp16)** and - **1.72–1.86× (bf16)** on the D64 path alone, matching the independent cross-part +**1.72–1.86× (bf16)** on the D64 path alone, matching the independent cross-part figure recorded when the lever was adopted. A benchmark that hand-builds its spec will silently miss a lever like this; building it from the dispatch factory is what keeps the measured config and the shipped config the same thing. @@ -78,7 +78,7 @@ faster than the fixed overhead. Two things worth knowing about how to read the t forcing the comparison onto a fallback vehicle; it builds now, so the baseline got faster while dense did not move. The ratio fell; nothing regressed. -D64 remains the VGPR-bound regime described under *Problem category* — `waves_per_eu` +D64 remains the VGPR-bound regime described under **Problem category** — `waves_per_eu` reaches 2 WG/CU for bf16-D64 but not fp16-D64 — so a `waves_per_eu` re-tune stays a named follow-up, now as upside rather than as a fix for a deficit. @@ -153,14 +153,14 @@ memory path. Two consequences worth stating explicitly, because both were mis-read at some point in this port: -- The runbook's bottleneck decision tree short-circuits on low occupancy *first*, so +- The runbook's bottleneck decision tree short-circuits on low occupancy **first**, so the LDS-bound branch is structurally unreachable here regardless of conflict rate. - `LDSBankConflict` counts conflicts **per LDS-active cycle**, so conflict-free V - *raises* the rate while *lowering* the cost. Reading that rate as a bound is the trap +**raises** the rate while **lowering** the cost. Reading that rate as a bound is the trap that kept the LDS-bound framing alive. The dominant remaining lever is therefore **occupancy** — getting a second workgroup -resident at D128, which needs an LDS cut and a register-floor cut *together* — plus +resident at D128, which needs an LDS cut and a register-floor cut **together** — plus grid shape at small sequence lengths. Per `AGENTS.md`, measured counters, utilisation and per-lever deltas are recorded outside the repo: see the optimization plan for this port and the protected results page. @@ -191,7 +191,8 @@ Measurements, register usage per toolchain, and methodology are in the | Lever | Config | Mechanism | Verdict | |---|---|---|---| | Conflict-free V at **bf16 D128** (`use_cfvst=True`) | bf16 D128 standard causal | same `perm_b32` store transpose as fp16 D128; fills the `waves_per_eu=2` VGPR budget with little headroom on the `.1k` MFMA schedule | **opt-in** — not a default; the earlier spill reading does not reproduce for CFVST alone | -| K-only LDS double buffer + V register prefetch (`lds_num_buffers=2`) | bf16 D128 CFVST standard causal, `block_n=64` only | two K LDS slots and one V LDS slot; the next V tile is carried in registers across softmax/PV, so it fits 64 KB LDS at `block_n=64` | **opt-in** — not a default; can spill depending on toolchain and grid. Gated off at `block_n=32`, where the second K slot lowers occupancy at `block_m=128` and no measurement exists | +| K-only LDS double buffering (`lds_num_buffers=2`) | bf16 D128 CFVST standard causal, `block_n=64` only | allocates two K LDS slots while keeping one V LDS slot | **opt-in** — independent of V register prefetch; can spill depending on toolchain and grid. Not validated at `block_n=32` | +| V register prefetch (`v_reg_prefetch=True`) | bf16 D128 CFVST standard causal, `block_n=64` only | loads V for the next tile into registers before the current PV; V LDS stays single-buffered | **opt-in** — independent of K LDS buffering; can also be combined with `lds_num_buffers=2` | ### Evaluated and rejected @@ -204,11 +205,11 @@ Measurements, register usage per toolchain, and methodology are in the | Drop K/V LDS pads to reach 2 WG/CU at D128 | D128 stays 1 WG/CU even unpadded (register floor co-limits), so it only reintroduces bank conflicts — **catastrophic** | | `block_n=32` (all configs) | halving the KV tile doubles the tile/grid count; the extra loop and barrier overhead outweighs the LDS relief. Marginally positive on **bf16 D128** but **part-dependent**, so not wired — it would need a CU-count-aware policy | | `iglp_opt` (`Gfx942AttentionDenseSpec.iglp`) | resource- and performance-neutral cross-part: the canned GEMM interleave does not match this loop, which is barrier-rendezvous-bound. Kept as a default-off knob | -| Smaller `BLOCK_M` | a *fully filled* grid at `BLOCK_M=64` measured **slower** than a one-third-filled one at 256 — 2 waves/CTA leaves half the CU's matrix cores unreachable at 1 CTA/CU. `BLOCK_M=128` is a small win on two configs only; not promoted | +| Smaller `BLOCK_M` | a **fully filled** grid at `BLOCK_M=64` measured **slower** than a one-third-filled one at 256 — 2 waves/CTA leaves half the CU's matrix cores unreachable at 1 CTA/CU. `BLOCK_M=128` is a small win on two configs only; not promoted | | cfvst load/store chunking | does not bound register pressure: the later chunks' loads carry no dependency on the earlier ones, so LLVM hoists them above the intervening full `s_waitcnt(vmcnt=0)`, which then covers them anyway | -| PV-only `s_setprio` | **proven-negative** on gfx942 `attention_tiled_2d`. Note this is one lever in one placement — it is *not* a verdict on the scheduling-intrinsic family, and hand-written `sched_group_barrier` remains open | +| PV-only `s_setprio` | **proven-negative** on gfx942 `attention_tiled_2d`. Note this is one lever in one placement — it is **not** a verdict on the scheduling-intrinsic family, and hand-written `sched_group_barrier` remains open | | Diagonal two-phase causal peel | **proven-negative** on gfx942 `attention_tiled_2d`; gated on the bound shifting to compute, which it has not | -| partial-vmcnt software prefetch | N/A on the default build — it is a double-buffering lever and the default build is single-buffered (NBUF=1), so there is no prefetch to partially overlap. A full K+V NBUF=2 fits 64 KB LDS at D128 only at `block_n=32`, which the `block_n=32` row above closes by measurement. The opt-in `lds_num_buffers=2` path (see the Opt-in table) doubles only K at `block_n=64` and waits with a full `vmcnt(0)`, not a partial one | +| partial-vmcnt software prefetch | N/A on the default build — it is a double-buffering lever and the default build is single-buffered (NBUF=1), so there is no prefetch to partially overlap. A full K+V NBUF=2 fits 64 KB LDS at D128 only at `block_n=32`, which the `block_n=32` row above closes by measurement. The opt-in `lds_num_buffers=2` path doubles only K at `block_n=64`; `v_reg_prefetch=True` independently brings next-tile V into registers. Both paths wait with full `vmcnt(0)`, not a partial one | ## Bench @@ -228,6 +229,6 @@ no GPU): re-run it after any lever change to confirm 0 spill and to see which re is the occupancy limiter. Per AGENTS.md, **absolute measured throughput lives only in the protected results page, -never in the repo.** The relative gains under *Status* are ratios, which is the form that +never in the repo.** The relative gains under **Status** are ratios, which is the form that can be stated here; every TFLOP/s, counter and per-lever magnitude behind them is on the -protected page. +protected page. \ No newline at end of file diff --git a/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx942/attention/prefill/gfx942_dense_cfvst_kdb_vprefetch_case_study.md b/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx942/attention/prefill/gfx942_dense_cfvst_kdb_vprefetch_case_study.md index 41fe54f7726f..ef92be703b6a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx942/attention/prefill/gfx942_dense_cfvst_kdb_vprefetch_case_study.md +++ b/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx942/attention/prefill/gfx942_dense_cfvst_kdb_vprefetch_case_study.md @@ -1,6 +1,6 @@ # gfx942 bf16 D128 causal prefill: CFVST, K LDS double buffering, and V register prefetch -Measured scope: the opt-in `use_cfvst=True`, `lds_num_buffers=2` configuration of the +Measured scope: the opt-in `use_cfvst=True`, `lds_num_buffers=2`, `v_reg_prefetch=True` configuration of the gfx942 dense prefill attention kernel, bf16, D128, causal, on gfx942 with 304 CUs. Baseline: the `baseline` arm of the ablation harness, compiled in the same process from the benchmarked source commit (see @@ -9,7 +9,7 @@ latency at the 12 measured configurations listed under Results. ## Introduction -This optimization enables CFVST and implements K LDS Double Buffering and V register pre-fetching on D128 bf16 causal prefill attention gfx942 kernels. At the 12 measured configurations it lowers latency by 10.1-16.3%. +This optimization enables CFVST and implements K LDS Double Buffering and V register pre-fetching on D128 bf16 causal prefill attention gfx942 kernels. At the 12 measured configurations it lowers latency by 10.74–15.58%. ## Problem @@ -18,7 +18,7 @@ Benchmarking, profiling and tracing highlighted bottlenecks in: 2. Awkward allocation of LDS per WG (33KB). Max limit per CU is 64KB. This limits WG occupancy in CU to 1 while leaving 31KB of LDS unused. 3. Stalls at sync point for loading K and V tiles before MFMA. -The concerned optimization points were V LDS layout, register usage, LDS usage, software pipelining. +The concerned optimization points were V LDS layout, register usage, LDS usage, software pipelining. ## Step-0 Lever Sweep @@ -102,7 +102,6 @@ At every stage, the sweep rejects unsupported/duplicate effective specs, numeric These are the best **final A/B-tested** configurations, compared with each workload's production-dispatch baseline. Each finalist passed a BF16-output-versus-FP32 causal SDPA numerical check (`atol=rtol=1e-2`) before timing. The timing statistic is the median of five alternating paired baseline/candidate rounds, with 50 timed launches per round. - | B | S | Hkv | Speedup | Latency reduction | Paired min–max | Baseline kernel | Winning kernel | |---:|---:|---:|---:|---:|---|---|---| | 1 | 1024 | 8 | **1.66346×** | **39.88%** | 1.64369–1.66779× | `rocke_attention_dense_d128_bn64_bf16_causal_lazyrs_gfx942_wpe2` | `rocke_attention_dense_d128_bn32_bf16_bm128_causal_lazyrs_gfx942_wpe4_krowpad4_e2f1_dsplit` | @@ -140,7 +139,7 @@ compiled by ROCm 7.1 comgr, and the `llvm23` IR flavor compiled by the ROCm 10 S |---|---:|---:|---:|---:|---:|---:| | bf16 default (CFVST off) | 243 | 0 | 0 | 219 | 0 | 0 | | bf16 `use_cfvst=True` | 256 | 0 | 0 | 234 | 0 | 0 | -| + `lds_num_buffers=2` | 256 | 4 | 20 | 230 | 0 | 0 | +| + `lds_num_buffers=2`, `v_reg_prefetch=True` | 256 | 4 | 20 | 230 | 0 | 0 | | + `emit_lse=True` | 256 | 5 | 24 | 230 | 0 | 0 | | + `persistent=True`, `num_persistent=304` | 256 | 16 | 68 | 256 | 7 | 32 | @@ -174,12 +173,12 @@ Analysis of stalls using threadtrace found a large number of waiting at sync poi | bf16 | 16 | 4096 | 32 | 16 | 128 | 1.1279x | 11.34% | 10.87% | 11.66% | | bf16 | 16 | 8192 | 32 | 8 | 128 | 1.1598x | 13.78% | 12.75% | 14.02% | -** Not a numerical error. A repeated 1st measurement anomaly. +** Not a numerical error. A repeated 1st measurement anomaly. #### Ablation studies of optimizations We performed ablation study to isolate the optimizations and understand its individual contributions. -| dtype | B | S | Hq | Hkv | D | CFVST vs base | CFVST +KDB vs CFVST | CFVST +VPF vs CFVST | CFSVT +KDB +VPF vs CFVST | CFSVT +KDB +VPF vs base | +| dtype | B | S | Hq | Hkv | D | CFVST vs base | CFVST +KDB vs CFVST | CFVST +VPF vs CFVST | CFVST +KDB +VPF vs CFVST | CFVST +KDB +VPF vs base | |------:|---:|------:|---:|----:|----:|----------------:|----------------:|----------------:|----------------:|----------------:| | bf16 | 1 | 1024 | 32 | 8 | 128 | 3.57% | -0.16% | 2.13% | 7.44% | 10.74% | | bf16 | 1 | 1024 | 32 | 16 | 128 | 4.55% | 0.12% | 3.19% | 7.79% | 11.99% | @@ -197,7 +196,6 @@ We performed ablation study to isolate the optimizations and understand its indi ## Results and Ablation Methodology The complete experiment can be summarized as follows: - For each of the 12 BF16 D128 causal attention shapes: 1. Start one ROCKE/PyTorch process using a single ROCm software stack. @@ -238,7 +236,6 @@ For each of the 12 BF16 D128 causal attention shapes: **Correctness coverage:** Each shape undergoes five initial correctness checks and 25 post-batch checks. Across 12 shapes, this gives **360 checks**. Post-batch validation checks the final output of each 50-launch batch, not every individual launch. - 20. Collect five round-level latency samples for each variant. 21. Compute the median latency independently for each variant: @@ -306,14 +303,19 @@ The five-way ablation was measured on gfx942 with 304 CUs, using 12 BF16 D128 ca **Benchmark harness:** `library/benchmarks/gfx942/attention/prefill/bench_rocke_5way_specs.py` **Kernel source:** `library/kernels/gfx942/attention_dense.py` -#### Reproducability +#### Reproducibility -Create gpu-bench/ at root. Clone branch into it. Below setup.sh sets up python env for rocke. Store it in -```bash -dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill/rocke_vs_aiter/setup.sh -``` -```bash +The following two Bash blocks contain the full contents of `setup.sh` and +`run_5way.sh`. Save them as separate files in +`dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill/`. +The scripts assume the repository is cloned at `/root/gpu-bench/rocm-libraries` +(`GPU_BENCH_ROOT` and `ROCM_REPO` can override this in `run_5way.sh`). +Run each script **as a file**, rather than pasting its individual commands into +an interactive shell; `run_5way.sh` locates the benchmark using its own path. + +**`setup.sh`** — install the ROCm Python environment and rocKE: +```bash #!/usr/bin/env bash set -Eeuo pipefail @@ -398,17 +400,17 @@ echo echo "ROCKE setup complete." echo "ROCKE env: source $ENV/rocke_env.sh" ``` -Run setup.sh + +Run the setup script from its directory: + ```bash -cd /root/gpu-bench/rocm-libraries/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill/ +cd /root/gpu-bench/rocm-libraries/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill chmod +x setup.sh ./setup.sh ``` -Store below run_5way.sh in -```bash -dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill/rocke_vs_aiter/run_5way.sh -``` +**`run_5way.sh`** — run the five-way benchmark and write its results: + ```bash #!/usr/bin/env bash set -Eeuo pipefail @@ -447,7 +449,7 @@ export ROCKE_COMGR_LIB="$CORE_LIB/libamd_comgr.so.3" export ROCKE_HIP_LIB="$CORE_LIB/libamdhip64.so.7" export LD_LIBRARY_PATH="$CORE_LIB" unset LD_PRELOAD -export PYTHONPATH="$ROCKE_ROOT/library:$ROCKE_ROOT/platform/Python:$ROCKE_ROOT${PYTHONPATH:+:$PYTHONPATH}" +export PYTHONPATH="$ROCKE_ROOT/library:$ROCKE_ROOT/platform/python:$ROCKE_ROOT${PYTHONPATH:+:$PYTHONPATH}" # One shell launch -> one Python process -> one current HIP stream. exec python "$SCRIPT_DIR/bench_rocke_5way_specs.py" \ @@ -455,14 +457,13 @@ exec python "$SCRIPT_DIR/bench_rocke_5way_specs.py" \ --out-dir "$OUT_DIR" \ "$@" ``` -Run run_5way.sh + +Run the benchmark from the same directory: + ```bash -cd /root/gpu-bench/rocm-libraries/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill/ +cd /root/gpu-bench/rocm-libraries/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill chmod +x run_5way.sh ./run_5way.sh ``` -Results available in results/ - ---- - +Results are written to `results/5way_specs/` by default. \ No newline at end of file