test(rocke): gfx942 dense benchmark and documentation fixes

dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill/bench_rocke_5way_specs.py

@@ -111,19 +111,17 @@ def locate_rocke(explicit: str | None) -> Path:
     starts = []
     if explicit:
         starts.append(Path(explicit).expanduser())
-    if os.environ.get("ROCKE_ROOT"):
-        starts.append(Path(os.environ["ROCKE_ROOT"]).expanduser())
     starts.extend((Path.cwd(), Path(__file__).resolve().parent))
     for start in starts:
         for path in (start, *start.parents):
             for candidate in (path, path / "dnn-providers/hip-kernel-provider/rocke"):
                 if (candidate / "library/kernels/gfx942/attention_dense.py").is_file():
                     return candidate.resolve()
-    raise SystemExit("ROCKE root not found: set --rocke-root or ROCKE_ROOT")
+    raise SystemExit("ROCKE root not found: pass --rocke-root")
 
 
 def add_module_paths(rocke: Path) -> None:
-    for p in (rocke / "library", rocke / "platform/Python", rocke):
+    for p in (rocke / "library", rocke / "platform/python", rocke):
         if str(p) not in sys.path:
             sys.path.insert(0, str(p))
 

dnn-providers/hip-kernel-provider/rocke/library/builders/gfx942/attention/prefill/README.md

@@ -68,7 +68,7 @@ faster than the fixed overhead. Two things worth knowing about how to read the t
   earlier comparison, run before the pad was adopted, measured D64 at −17 % (fp16,
   short) to +29 %. Re-running with the pad in place is what removed the regression: an
   explicit ON/OFF A/B in the same harness puts it at **1.88–2.13× (fp16)** and
-  **1.72–1.86× (bf16)** on the D64 path alone, matching the independent cross-part
+**1.72–1.86× (bf16)** on the D64 path alone, matching the independent cross-part
   figure recorded when the lever was adopted. A benchmark that hand-builds its spec
   will silently miss a lever like this; building it from the dispatch factory is what
   keeps the measured config and the shipped config the same thing.
@@ -78,7 +78,7 @@ faster than the fixed overhead. Two things worth knowing about how to read the t
   forcing the comparison onto a fallback vehicle; it builds now, so the baseline got
   faster while dense did not move. The ratio fell; nothing regressed.
 
-D64 remains the VGPR-bound regime described under *Problem category* — `waves_per_eu`
+D64 remains the VGPR-bound regime described under **Problem category** — `waves_per_eu`
 reaches 2 WG/CU for bf16-D64 but not fp16-D64 — so a `waves_per_eu` re-tune stays a
 named follow-up, now as upside rather than as a fix for a deficit.
 
@@ -153,14 +153,14 @@ memory path.
 Two consequences worth stating explicitly, because both were mis-read at some point in
 this port:
 
-- The runbook's bottleneck decision tree short-circuits on low occupancy *first*, so
+- The runbook's bottleneck decision tree short-circuits on low occupancy **first**, so
   the LDS-bound branch is structurally unreachable here regardless of conflict rate.
 - `LDSBankConflict` counts conflicts **per LDS-active cycle**, so conflict-free V
-  *raises* the rate while *lowering* the cost. Reading that rate as a bound is the trap
+**raises** the rate while **lowering** the cost. Reading that rate as a bound is the trap
   that kept the LDS-bound framing alive.
 
 The dominant remaining lever is therefore **occupancy** — getting a second workgroup
-resident at D128, which needs an LDS cut and a register-floor cut *together* — plus
+resident at D128, which needs an LDS cut and a register-floor cut **together** — plus
 grid shape at small sequence lengths. Per `AGENTS.md`, measured counters, utilisation
 and per-lever deltas are recorded outside the repo: see the optimization plan for this
 port and the protected results page.
@@ -191,7 +191,8 @@ Measurements, register usage per toolchain, and methodology are in the
 | Lever | Config | Mechanism | Verdict |
 |---|---|---|---|
 | Conflict-free V at **bf16 D128** (`use_cfvst=True`) | bf16 D128 standard causal | same `perm_b32` store transpose as fp16 D128; fills the `waves_per_eu=2` VGPR budget with little headroom on the `.1k` MFMA schedule | **opt-in** — not a default; the earlier spill reading does not reproduce for CFVST alone |
-| K-only LDS double buffer + V register prefetch (`lds_num_buffers=2`) | bf16 D128 CFVST standard causal, `block_n=64` only | two K LDS slots and one V LDS slot; the next V tile is carried in registers across softmax/PV, so it fits 64 KB LDS at `block_n=64` | **opt-in** — not a default; can spill depending on toolchain and grid. Gated off at `block_n=32`, where the second K slot lowers occupancy at `block_m=128` and no measurement exists |
+| K-only LDS double buffering (`lds_num_buffers=2`) | bf16 D128 CFVST standard causal, `block_n=64` only | allocates two K LDS slots while keeping one V LDS slot | **opt-in** — independent of V register prefetch; can spill depending on toolchain and grid. Not validated at `block_n=32` |
+| V register prefetch (`v_reg_prefetch=True`) | bf16 D128 CFVST standard causal, `block_n=64` only | loads V for the next tile into registers before the current PV; V LDS stays single-buffered | **opt-in** — independent of K LDS buffering; can also be combined with `lds_num_buffers=2` |
 
 ### Evaluated and rejected
 
@@ -204,11 +205,11 @@ Measurements, register usage per toolchain, and methodology are in the
 | Drop K/V LDS pads to reach 2 WG/CU at D128 | D128 stays 1 WG/CU even unpadded (register floor co-limits), so it only reintroduces bank conflicts — **catastrophic** |
 | `block_n=32` (all configs) | halving the KV tile doubles the tile/grid count; the extra loop and barrier overhead outweighs the LDS relief. Marginally positive on **bf16 D128** but **part-dependent**, so not wired — it would need a CU-count-aware policy |
 | `iglp_opt` (`Gfx942AttentionDenseSpec.iglp`) | resource- and performance-neutral cross-part: the canned GEMM interleave does not match this loop, which is barrier-rendezvous-bound. Kept as a default-off knob |
-| Smaller `BLOCK_M` | a *fully filled* grid at `BLOCK_M=64` measured **slower** than a one-third-filled one at 256 — 2 waves/CTA leaves half the CU's matrix cores unreachable at 1 CTA/CU. `BLOCK_M=128` is a small win on two configs only; not promoted |
+| Smaller `BLOCK_M` | a **fully filled** grid at `BLOCK_M=64` measured **slower** than a one-third-filled one at 256 — 2 waves/CTA leaves half the CU's matrix cores unreachable at 1 CTA/CU. `BLOCK_M=128` is a small win on two configs only; not promoted |
 | cfvst load/store chunking | does not bound register pressure: the later chunks' loads carry no dependency on the earlier ones, so LLVM hoists them above the intervening full `s_waitcnt(vmcnt=0)`, which then covers them anyway |
-| PV-only `s_setprio` | **proven-negative** on gfx942 `attention_tiled_2d`. Note this is one lever in one placement — it is *not* a verdict on the scheduling-intrinsic family, and hand-written `sched_group_barrier` remains open |
+| PV-only `s_setprio` | **proven-negative** on gfx942 `attention_tiled_2d`. Note this is one lever in one placement — it is **not** a verdict on the scheduling-intrinsic family, and hand-written `sched_group_barrier` remains open |
 | Diagonal two-phase causal peel | **proven-negative** on gfx942 `attention_tiled_2d`; gated on the bound shifting to compute, which it has not |
-| partial-vmcnt software prefetch | N/A on the default build — it is a double-buffering lever and the default build is single-buffered (NBUF=1), so there is no prefetch to partially overlap. A full K+V NBUF=2 fits 64 KB LDS at D128 only at `block_n=32`, which the `block_n=32` row above closes by measurement. The opt-in `lds_num_buffers=2` path (see the Opt-in table) doubles only K at `block_n=64` and waits with a full `vmcnt(0)`, not a partial one |
+| partial-vmcnt software prefetch | N/A on the default build — it is a double-buffering lever and the default build is single-buffered (NBUF=1), so there is no prefetch to partially overlap. A full K+V NBUF=2 fits 64 KB LDS at D128 only at `block_n=32`, which the `block_n=32` row above closes by measurement. The opt-in `lds_num_buffers=2` path doubles only K at `block_n=64`; `v_reg_prefetch=True` independently brings next-tile V into registers. Both paths wait with full `vmcnt(0)`, not a partial one |
 
 ## Bench
 
@@ -228,6 +229,6 @@ no GPU): re-run it after any lever change to confirm 0 spill and to see which re
 is the occupancy limiter.
 
 Per AGENTS.md, **absolute measured throughput lives only in the protected results page,
-never in the repo.** The relative gains under *Status* are ratios, which is the form that
+never in the repo.** The relative gains under **Status** are ratios, which is the form that
 can be stated here; every TFLOP/s, counter and per-lever magnitude behind them is on the
-protected page.
+protected page.

dnn-providers/hip-kernel-provider/rocke/library/builders/gfx942/attention/prefill/gfx942_dense_cfvst_kdb_vprefetch_case_study.md

@@ -1,6 +1,6 @@
 # gfx942 bf16 D128 causal prefill: CFVST, K LDS double buffering, and V register prefetch
 
-Measured scope: the opt-in `use_cfvst=True`, `lds_num_buffers=2` configuration of the
+Measured scope: the opt-in `use_cfvst=True`, `lds_num_buffers=2`, `v_reg_prefetch=True` configuration of the
 gfx942 dense prefill attention kernel, bf16, D128, causal, on gfx942 with 304 CUs.
 Baseline: the `baseline` arm of the ablation harness, compiled in the same process from
 the benchmarked source commit (see
@@ -9,7 +9,7 @@ latency at the 12 measured configurations listed under Results.
 
 ## Introduction
 
-This optimization enables CFVST and implements K LDS Double Buffering and V register pre-fetching on D128 bf16 causal prefill attention gfx942 kernels. At the 12 measured configurations it lowers latency by 10.1-16.3%. 
+This optimization enables CFVST and implements K LDS Double Buffering and V register pre-fetching on D128 bf16 causal prefill attention gfx942 kernels. At the 12 measured configurations it lowers latency by 10.74–15.58%.
 
 ## Problem
 
@@ -18,7 +18,7 @@ Benchmarking, profiling and tracing highlighted bottlenecks in:
 2. Awkward allocation of LDS per WG (33KB). Max limit per CU is 64KB. This limits WG occupancy in CU to 1 while leaving 31KB of LDS unused.
 3. Stalls at sync point for loading K and V tiles before MFMA.
 
-The concerned optimization points were V LDS layout, register usage, LDS usage, software pipelining. 
+The concerned optimization points were V LDS layout, register usage, LDS usage, software pipelining.
 
 ## Step-0 Lever Sweep
 
@@ -102,7 +102,6 @@ At every stage, the sweep rejects unsupported/duplicate effective specs, numeric
 
 These are the best **final A/B-tested** configurations, compared with each workload's production-dispatch baseline. Each finalist passed a BF16-output-versus-FP32 causal SDPA numerical check (`atol=rtol=1e-2`) before timing. The timing statistic is the median of five alternating paired baseline/candidate rounds, with 50 timed launches per round.
 
-
 | B | S | Hkv | Speedup | Latency reduction | Paired min–max | Baseline kernel | Winning kernel |
 |---:|---:|---:|---:|---:|---|---|---|
 | 1 | 1024 | 8 | **1.66346×** | **39.88%** | 1.64369–1.66779× | `rocke_attention_dense_d128_bn64_bf16_causal_lazyrs_gfx942_wpe2` | `rocke_attention_dense_d128_bn32_bf16_bm128_causal_lazyrs_gfx942_wpe4_krowpad4_e2f1_dsplit` |
@@ -140,7 +139,7 @@ compiled by ROCm 7.1 comgr, and the `llvm23` IR flavor compiled by the ROCm 10 S
 |---|---:|---:|---:|---:|---:|---:|
 | bf16 default (CFVST off) | 243 | 0 | 0 | 219 | 0 | 0 |
 | bf16 `use_cfvst=True` | 256 | 0 | 0 | 234 | 0 | 0 |
-| + `lds_num_buffers=2` | 256 | 4 | 20 | 230 | 0 | 0 |
+| + `lds_num_buffers=2`, `v_reg_prefetch=True` | 256 | 4 | 20 | 230 | 0 | 0 |
 | + `emit_lse=True` | 256 | 5 | 24 | 230 | 0 | 0 |
 | + `persistent=True`, `num_persistent=304` | 256 | 16 | 68 | 256 | 7 | 32 |
 
@@ -174,12 +173,12 @@ Analysis of stalls using threadtrace found a large number of waiting at sync poi
 | bf16 | 16 | 4096 | 32 | 16 | 128 | 1.1279x | 11.34% | 10.87% | 11.66% |
 | bf16 | 16 | 8192 | 32 | 8 | 128 | 1.1598x | 13.78% | 12.75% | 14.02% |
 
-** Not a numerical error. A repeated 1st measurement anomaly. 
+** Not a numerical error. A repeated 1st measurement anomaly.
 
 #### Ablation studies of optimizations
 We performed ablation study to isolate the optimizations and understand its individual contributions.
 
-| dtype | B | S | Hq | Hkv | D | CFVST vs base |  CFVST +KDB vs CFVST | CFVST +VPF vs CFVST | CFSVT +KDB +VPF vs CFVST | CFSVT +KDB +VPF vs base |
+| dtype | B | S | Hq | Hkv | D | CFVST vs base |  CFVST +KDB vs CFVST | CFVST +VPF vs CFVST | CFVST +KDB +VPF vs CFVST | CFVST +KDB +VPF vs base |
 |------:|---:|------:|---:|----:|----:|----------------:|----------------:|----------------:|----------------:|----------------:|
 | bf16 | 1 | 1024 | 32 | 8 | 128 | 3.57% | -0.16% | 2.13% | 7.44% | 10.74% |
 | bf16 | 1 | 1024 | 32 | 16 | 128 | 4.55% | 0.12% | 3.19% | 7.79% | 11.99% |
@@ -197,7 +196,6 @@ We performed ablation study to isolate the optimizations and understand its indi
 ## Results and Ablation Methodology
 The complete experiment can be summarized as follows:
 
-
 For each of the 12 BF16 D128 causal attention shapes:
 
 1. Start one ROCKE/PyTorch process using a single ROCm software stack.
@@ -238,7 +236,6 @@ For each of the 12 BF16 D128 causal attention shapes:
 
 **Correctness coverage:** Each shape undergoes five initial correctness checks and 25 post-batch checks. Across 12 shapes, this gives **360 checks**. Post-batch validation checks the final output of each 50-launch batch, not every individual launch.
 
-
 20. Collect five round-level latency samples for each variant.
 21. Compute the median latency independently for each variant:
 
@@ -306,14 +303,19 @@ The five-way ablation was measured on gfx942 with 304 CUs, using 12 BF16 D128 ca
 **Benchmark harness:** `library/benchmarks/gfx942/attention/prefill/bench_rocke_5way_specs.py`
 **Kernel source:** `library/kernels/gfx942/attention_dense.py`
 
-#### Reproducability
+#### Reproducibility
 
-Create gpu-bench/ at root. Clone branch into it. Below setup.sh sets up python env for rocke. Store it in 
-```bash
-dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill/rocke_vs_aiter/setup.sh
-```
-```bash
+The following two Bash blocks contain the full contents of `setup.sh` and
+`run_5way.sh`. Save them as separate files in
+`dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill/`.
+The scripts assume the repository is cloned at `/root/gpu-bench/rocm-libraries`
+(`GPU_BENCH_ROOT` and `ROCM_REPO` can override this in `run_5way.sh`).
+Run each script **as a file**, rather than pasting its individual commands into
+an interactive shell; `run_5way.sh` locates the benchmark using its own path.
+
+**`setup.sh`** — install the ROCm Python environment and rocKE:
 
+```bash
 #!/usr/bin/env bash
 set -Eeuo pipefail
 
@@ -398,17 +400,17 @@ echo
 echo "ROCKE setup complete."
 echo "ROCKE env: source $ENV/rocke_env.sh"
 ```
-Run setup.sh
+
+Run the setup script from its directory:
+
 ```bash
-cd /root/gpu-bench/rocm-libraries/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill/
+cd /root/gpu-bench/rocm-libraries/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill
 chmod +x setup.sh
 ./setup.sh
 ```
 
-Store below run_5way.sh in
-```bash
-dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill/rocke_vs_aiter/run_5way.sh
-```
+**`run_5way.sh`** — run the five-way benchmark and write its results:
+
 ```bash
 #!/usr/bin/env bash
 set -Eeuo pipefail
@@ -447,22 +449,21 @@ export ROCKE_COMGR_LIB="$CORE_LIB/libamd_comgr.so.3"
 export ROCKE_HIP_LIB="$CORE_LIB/libamdhip64.so.7"
 export LD_LIBRARY_PATH="$CORE_LIB"
 unset LD_PRELOAD
-export PYTHONPATH="$ROCKE_ROOT/library:$ROCKE_ROOT/platform/Python:$ROCKE_ROOT${PYTHONPATH:+:$PYTHONPATH}"
+export PYTHONPATH="$ROCKE_ROOT/library:$ROCKE_ROOT/platform/python:$ROCKE_ROOT${PYTHONPATH:+:$PYTHONPATH}"
 
 # One shell launch -> one Python process -> one current HIP stream.
 exec python "$SCRIPT_DIR/bench_rocke_5way_specs.py" \
   --rocke-root "$ROCKE_ROOT" \
   --out-dir "$OUT_DIR" \
   "$@"
 ```
-Run run_5way.sh
+
+Run the benchmark from the same directory:
+
 ```bash
-cd /root/gpu-bench/rocm-libraries/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill/
+cd /root/gpu-bench/rocm-libraries/dnn-providers/hip-kernel-provider/rocke/library/benchmarks/gfx942/attention/prefill
 chmod +x run_5way.sh
 ./run_5way.sh
 ```
-Results available in results/
-
----
-
 
+Results are written to `results/5way_specs/` by default.

dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx942/attention_dense.py

@@ -890,15 +890,6 @@ def _use_vpf(spec: "Gfx942AttentionDenseSpec") -> bool:
     return spec.v_reg_prefetch
 
 
-def _use_kdb_vpf(spec: "Gfx942AttentionDenseSpec") -> bool:
-    """Combined variant predicate kept for existing external callers.
-
-    Neither knob implies the other. The kernel's allocation, scheduling, and
-    naming use ``_use_kdb`` and ``_use_vpf`` independently.
-    """
-    return _use_kdb(spec) and _use_vpf(spec)
-
-
 def _v_swizzle_width(block_n: int) -> int:
     """V^T row width (elements) the XOR bank-conflict swizzle needs: the smallest pow2
     >= max(64, block_n). SINGLE source of truth -- the derived pad (:func:`_v_row_pad`),