From 3e5da5c0b400346c794dc61b30e2e32f00127155 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Mon, 12 Oct 2026 02:33:01 -0400 Subject: [PATCH 1/3] perf(rocke): measured per-mode defaults for gfx950 GDN/KDA decode auto on the gfx950 GDN/KDA decode dispatcher now runs one static configuration per case -- gate kind x fused (fuse_conv or fuse_out_norm) x state width -- a registered tile plus a knob stack on top of its base, measured cold on gfx950 at D128 (fused cases with both fused flags on). Batch, head count and batch * num_v_heads never select it. - dispatch/gdn/axes.py: AUTO_DEFAULTS, the eight (tile, knobs) rows as data, with the selection rule; AUTO_STATE_ROW (an f16 state was not measured and runs the bf16 row). At import every row's knobs must be values of their axes. - dispatch/gdn/gfx950.py: auto_default() reads the case; auto on the case's tile resolves the stack through the tile's knob space, so the configuration is a registered one and its result's request (spec_id, tuning_id, tuning_knobs) replays it. A knob the request cannot read is dropped as for any pin; norm_gate_once drops with conv_once, since the kernel refuses it alone (a norm-only GDN request runs the rest of its row). Each gate kind's priority order is its auto tiles, then the rest in product order. When the case's tile is illegal (GDN head_k_dim 64 / 192) auto runs the first legal tile in that order at its base configuration. - rocke.dispatch.tuning: make_tuned_candidate takes an optional auto_knobs(req) beside route_auto: the knobs an auto request runs on that candidate, resolved as tuning_knobs. Empty runs the space's default; knobs that do not resolve where the default does raise with the reason instead of running another configuration; auto_knobs without route_auto is refused; a request that carries its own tuning pin (carries_tuning_pin: a tuning_id other than auto, or tuning_knobs) resolves it as given. Other families are unchanged. - KDA fused f32 ships (4, 8, 1) with dpp_reduce, stream_rows, interleave_cols, conv_once and out_lds, an override of the rule's (16, 16, 1) pick chosen for batch 1 (the latency case); the row is the best-scoring (4, 8, 1) configuration, near-tied with the rule's pick. - KDA plain's work-keyed table (_TUNED_TILES_KDA, work_for, tile_for_work), GDN's single DEFAULT_TILE and the interim FUSED_DEFAULT_TILE are removed. tune.py drops its work-keying report, sweeps KDA tiles from the dispatched tile with every knob off, and labels the auto tile's base configuration as such; it no longer advises a tile from base-only timings. Pins are unchanged: a bare spec_id still runs the tile with every knob off, and every tuning_id / config_key is the same. Golden: no existing hash changes. Twenty-four auto_* cases are added: gate kind x plain / conv + norm x bf16 / f16 / f32 state, the configuration auto dispatches per case, and gate kind x conv only / norm only / norm only at GQA heads x bf16 / f32, the row less the knobs a one-flag request cannot read. Moving a default moves a hash. Tests: the shipped table as a literal, the same configuration at every batch, f16 -> bf16 row, scope drops for one fused flag (norm-only GDN included), the fallback over a broad request domain (DK != DV included), pins unchanged, tune.py's KDA sweep base has every knob off, the shared hook raises on an unresolved stack and never replaces a caller's pin (by knobs or by id alone), and every auto configuration (one-flag and GQA norm-only included) compiles with no scratch. On device, through dispatch against the fp32 reference: every case's auto configuration and the one-flag and GQA norm-only modes at bf16 and f32 state. Docs: ALGORITHM.md section 4.6 (table, shapes measured and not, rule, override, fallback), README, dsl_docs/instances/gdn.md and index.md, rocke/dispatch/ARCHITECTURE.md section 14 (auto_knobs), the GPU coverage map's KDA rows, and CASE_STUDY.md (the tiles it measured are named as the defaults of that time; its section 7 script runs again). --- .../rocke/docs/gpu-attention-test-coverage.md | 25 +- .../library/builders/gfx950/gdn/ALGORITHM.md | 151 ++++--- .../library/builders/gfx950/gdn/CASE_STUDY.md | 45 +- .../library/builders/gfx950/gdn/README.md | 58 +-- .../rocke/library/builders/gfx950/gdn/tune.py | 100 ++--- .../rocke/library/dispatch/gdn/__init__.py | 29 +- .../rocke/library/dispatch/gdn/axes.py | 127 +++++- .../rocke/library/dispatch/gdn/common.py | 25 +- .../rocke/library/dispatch/gdn/gfx950.py | 138 +++--- .../library/kernels/gfx950/gdn_decode.py | 9 +- .../dispatch/gdn/test_gfx950_registry.py | 14 +- .../tests/dispatch/gdn/test_gfx950_wiring.py | 419 +++++++++++------- .../dispatch/gdn/test_tuning_contract.py | 61 ++- .../golden/gdn_decode_gfx950_ir_sha256.json | 288 ++++++++++++ .../tests/test_gdn_decode_gfx950_numeric.py | 94 +++- .../library/tests/test_gdn_decode_golden.py | 48 +- .../tests/test_gdn_decode_load_order.py | 19 +- .../library/tests/test_gdn_decode_spec.py | 65 ++- .../library/tests/test_gdn_decode_tune.py | 66 +-- .../tests/test_kda_decode_benchmark.py | 4 +- .../tests/test_kda_decode_gfx950_numeric.py | 47 +- .../tests/test_kda_decode_resources.py | 22 +- .../rocke/platform/dsl_docs/instances/gdn.md | 35 +- .../platform/dsl_docs/instances/index.md | 10 +- .../python/rocke/dispatch/ARCHITECTURE.md | 26 +- .../python/rocke/dispatch/tuning/candidate.py | 54 ++- .../tuning/test_tuning_template.py | 41 ++ 27 files changed, 1377 insertions(+), 643 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/docs/gpu-attention-test-coverage.md b/dnn-providers/hip-kernel-provider/rocke/docs/gpu-attention-test-coverage.md index 368323f7ddfb..e0652c68d643 100644 --- a/dnn-providers/hip-kernel-provider/rocke/docs/gpu-attention-test-coverage.md +++ b/dnn-providers/hip-kernel-provider/rocke/docs/gpu-attention-test-coverage.md @@ -20,11 +20,14 @@ cases and three negative checks passed; the wider library GPU suite reported 84 passes and 134 skips. Published artifact packaging was separately verified at run `36897764926`. A source-declared shape is not necessarily selected, executed, or passed in CI. -One edit postdates that inventory: the `expected_spec_id` values of the -`test_kda_dispatch_band_launches_selected_kernel` rows use the tile-named KDA -spec ids (`kda_nw_wtk_bpv`) that later replaced the work-band names; -the tests and shapes are unchanged. GPU tests added after the inventory are not -listed here. +Two edits postdate that inventory, both to the KDA decode dispatch rows. The +`expected_spec_id` values use the tile-named KDA spec ids +(`kda_nw_wtk_bpv`) that replaced the work-band names. The work-keyed +tile table was later replaced by one static `auto` configuration per case: the +band test is now `test_kda_dispatch_launches_the_auto_kernel` and the tuned-tile +test `test_every_kda_auto_tile_is_correct`, with the `auto` tiles as their +shapes. Both also run with an `f32` state; those state-width rows are not +listed. Other GPU tests added after the inventory are not listed here. ## Reading this map @@ -425,12 +428,12 @@ not a rocKE kernel execution; it remains here to expose that distinction. | [test_kda_decode_gfx950_numeric.py](../library/tests/test_kda_decode_gfx950_numeric.py) :: test_kda_warp_tiled_matches_reference | gfx950 | 3 | 1 | 16 | 32 | 128 | 128 | bf16 | {"batch": 3}; output and updated state checked; state [pool,Hv,DV,DK] | Selected by library CTest file list | | [test_kda_decode_gfx950_numeric.py](../library/tests/test_kda_decode_gfx950_numeric.py) :: test_kda_warp_tiled_matches_reference | gfx950 | 16 | 1 | 16 | 32 | 128 | 128 | bf16 | {"batch": 16}; output and updated state checked; state [pool,Hv,DV,DK] | Selected by library CTest file list | | [test_kda_decode_gfx950_numeric.py](../library/tests/test_kda_decode_gfx950_numeric.py) :: test_kda_warp_tiled_matches_reference | gfx950 | 64 | 1 | 16 | 32 | 128 | 128 | bf16 | {"batch": 64}; output and updated state checked; state [pool,Hv,DV,DK] | Selected by library CTest file list | -| [test_kda_decode_gfx950_numeric.py](../library/tests/test_kda_decode_gfx950_numeric.py) :: test_every_kda_tuned_tile_is_correct | gfx950 | 16 | 1 | 16 | 32 | 128 | 128 | bf16 | {"batch": 16, "tile": [4, 16, 4]}; output and updated state checked; state [pool,Hv,DV,DK] | Selected by library CTest file list | -| [test_kda_decode_gfx950_numeric.py](../library/tests/test_kda_decode_gfx950_numeric.py) :: test_every_kda_tuned_tile_is_correct | gfx950 | 16 | 1 | 16 | 32 | 128 | 128 | bf16 | {"batch": 16, "tile": [1, 16, 4]}; output and updated state checked; state [pool,Hv,DV,DK] | Selected by library CTest file list | -| [test_kda_decode_gfx950_numeric.py](../library/tests/test_kda_decode_gfx950_numeric.py) :: test_every_kda_tuned_tile_is_correct | gfx950 | 16 | 1 | 16 | 32 | 128 | 128 | bf16 | {"batch": 16, "tile": [2, 16, 1]}; output and updated state checked; state [pool,Hv,DV,DK] | Selected by library CTest file list | -| [test_kda_decode_gfx950_numeric.py](../library/tests/test_kda_decode_gfx950_numeric.py) :: test_kda_dispatch_band_launches_selected_kernel | gfx950 | 1 | 1 | 32 | 32 | 128 | 128 | bf16 | {"batch": 1, "expected_spec_id": "kda_nw4_wtk16_bpv4", "expected_tile": [4, 16, 4]}; output and updated state checked; state [pool,Hv,DV,DK] | Selected by library CTest file list | -| [test_kda_decode_gfx950_numeric.py](../library/tests/test_kda_decode_gfx950_numeric.py) :: test_kda_dispatch_band_launches_selected_kernel | gfx950 | 8 | 1 | 32 | 32 | 128 | 128 | bf16 | {"batch": 8, "expected_spec_id": "kda_nw1_wtk16_bpv4", "expected_tile": [1, 16, 4]}; output and updated state checked; state [pool,Hv,DV,DK] | Selected by library CTest file list | -| [test_kda_decode_gfx950_numeric.py](../library/tests/test_kda_decode_gfx950_numeric.py) :: test_kda_dispatch_band_launches_selected_kernel | gfx950 | 32 | 1 | 32 | 32 | 128 | 128 | bf16 | {"batch": 32, "expected_spec_id": "kda_nw2_wtk16_bpv1", "expected_tile": [2, 16, 1]}; output and updated state checked; state [pool,Hv,DV,DK] | Selected by library CTest file list | +| [test_kda_decode_gfx950_numeric.py](../library/tests/test_kda_decode_gfx950_numeric.py) :: test_every_kda_auto_tile_is_correct | gfx950 | 16 | 1 | 16 | 32 | 128 | 128 | bf16 | {"batch": 16, "tile": [1, 16, 8]}; output and updated state checked; state [pool,Hv,DV,DK] | Selected by library CTest file list | +| [test_kda_decode_gfx950_numeric.py](../library/tests/test_kda_decode_gfx950_numeric.py) :: test_every_kda_auto_tile_is_correct | gfx950 | 16 | 1 | 16 | 32 | 128 | 128 | bf16 | {"batch": 16, "tile": [1, 16, 16]}; output and updated state checked; state [pool,Hv,DV,DK] | Selected by library CTest file list | +| [test_kda_decode_gfx950_numeric.py](../library/tests/test_kda_decode_gfx950_numeric.py) :: test_every_kda_auto_tile_is_correct | gfx950 | 16 | 1 | 16 | 32 | 128 | 128 | bf16 | {"batch": 16, "tile": [4, 8, 1]}; output and updated state checked; state [pool,Hv,DV,DK] | Selected by library CTest file list | +| [test_kda_decode_gfx950_numeric.py](../library/tests/test_kda_decode_gfx950_numeric.py) :: test_kda_dispatch_launches_the_auto_kernel | gfx950 | 1 | 1 | 32 | 32 | 128 | 128 | bf16 | {"batch": 1, "expected_spec_id": "kda_nw1_wtk16_bpv8", "expected_tile": [1, 16, 8]}; output and updated state checked; state [pool,Hv,DV,DK] | Selected by library CTest file list | +| [test_kda_decode_gfx950_numeric.py](../library/tests/test_kda_decode_gfx950_numeric.py) :: test_kda_dispatch_launches_the_auto_kernel | gfx950 | 8 | 1 | 32 | 32 | 128 | 128 | bf16 | {"batch": 8, "expected_spec_id": "kda_nw1_wtk16_bpv8", "expected_tile": [1, 16, 8]}; output and updated state checked; state [pool,Hv,DV,DK] | Selected by library CTest file list | +| [test_kda_decode_gfx950_numeric.py](../library/tests/test_kda_decode_gfx950_numeric.py) :: test_kda_dispatch_launches_the_auto_kernel | gfx950 | 32 | 1 | 32 | 32 | 128 | 128 | bf16 | {"batch": 32, "expected_spec_id": "kda_nw1_wtk16_bpv8", "expected_tile": [1, 16, 8]}; output and updated state checked; state [pool,Hv,DV,DK] | Selected by library CTest file list | | [test_kda_decode_gfx950_numeric.py](../library/tests/test_kda_decode_gfx950_numeric.py) :: test_kda_mha_shape_matches_reference | gfx950 | 8 | 1 | 32 | 32 | 128 | 128 | bf16 | {}; output and updated state checked; state [pool,Hv,DV,DK] | Selected by library CTest file list | | [test_kda_decode_gfx950_numeric.py](../library/tests/test_kda_decode_gfx950_numeric.py) :: test_precomputed_decay_matches_reference | gfx950 | 1 | 1 | 16 | 32 | 128 | 128 | bf16 | {"batch": 1}; output and updated state checked; state [pool,Hv,DV,DK] | Selected by library CTest file list | | [test_kda_decode_gfx950_numeric.py](../library/tests/test_kda_decode_gfx950_numeric.py) :: test_precomputed_decay_matches_reference | gfx950 | 16 | 1 | 16 | 32 | 128 | 128 | bf16 | {"batch": 16}; output and updated state checked; state [pool,Hv,DV,DK] | Selected by library CTest file list | diff --git a/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/gdn/ALGORITHM.md b/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/gdn/ALGORITHM.md index f578a79811fd..85b18c35fe47 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/gdn/ALGORITHM.md +++ b/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/gdn/ALGORITHM.md @@ -212,9 +212,9 @@ Three things the table pins down that the rest of the document assumes: - **Where these models land in the selection policies.** Both have `Hv = 32`. Prefill's `value_splits` bands on `BH = 32 × batch` (`≤64 → 8`, `≤128 → 2`, else `1`), so batches 1-2 get 8 splits, batches 3-4 get 2, and larger - batches run unsplit. GDN decode instead enumerates validator-approved - registry tiles and uses the documented static `(2, 16, 8)` priority for the - supported D128 deployment; batch changes the grid, not its tile. + batches run unsplit. GDN decode instead runs one static configuration per + case (§4.6; `(2, 16, 8)` for a 16-bit state, plain) on the supported D128 + deployment; batch changes the grid, not its configuration. This also fixes the scope of §2.3's reuse-over-fork argument. A target that keeps the gated delta rule but changes the gate's *formula* stays a `gate_kind`, not a fork. A target that changes `C`, @@ -387,11 +387,12 @@ One workgroup, one decode step, in emission order (warp-tiled path): Why loads first: the AMDGPU scheduler does not hoist a load above earlier math or across the `k_lane == 0` output-store branch, so emission order bounds how many separate load batches (each -ended by a `vmcnt` wait) a wave exposes. In the gfx950 code object, loads first brings the GDN -default tile and the KDA tuned tiles down to one or two batches. Every tile with `WTK ≤ 2`, most -with `WTK = 4`, and the fused tiles still split their loads into several batches: under register -pressure the compiler re-interleaves some loads with the math. These are static counts from the -code object, not hardware counters. +ended by a `vmcnt` wait) a wave exposes. In the gfx950 code object, loads first brings GDN +`(2,16,8)` and KDA `(4,16,4)`, `(1,16,4)`, `(2,16,1)` (the plain `auto` tiles when this order +landed) down to one or two batches. Every tile with `WTK ≤ 2`, most with `WTK = 4`, and the fused +tiles still split their loads into several batches: under register pressure the compiler +re-interleaves some loads with the math. These are static counts from the code object, not +hardware counters. What moved: only loads, plus (with `fuse_conv`) the `v` conv + SiLU, which now runs next to the `q`/`k` conv before the L2 norms instead of inside the step-8 row loop. Each value still goes @@ -434,8 +435,7 @@ fused cases that every measured compiler keeps there. `CASE_STUDY.md` has the pe each measured compiler and the command that regenerates them; re-run it when the compiler or the lowering changes. -Tile defaults (§4.6) were measured on the interleaved order and are re-measured when the defaults -are re-picked. +The §4.6 `auto` defaults were measured on this loads-first order. The simple path (`spec.simple`) keeps the interleaved order: q/k, norms, gates, dot, then the state row. @@ -502,10 +502,8 @@ This produces 180 stable tile identities per gate kind: `nw_wtk_bpv` for GDN and the same with a `kda_` prefix for KDA. `is_valid_spec()` is the only legality authority and admits 54 tiles for the default D128 shape. KDA accepts only `DK = DV = 128` today, so 54 of its 180 tiles are admissible -and the other 126 stay registered but unselectable until that limit is lifted. Production `auto` -deterministically prefers GDN's `(2, 16, 8)` whenever -legal; batch changes grid size, not GDN tile selection. A caller may pin an exact tile by its -`spec_id`. +and the other 126 stay registered but unselectable until that limit is lifted. A caller may pin an +exact tile by its `spec_id`. Each registered tile is a tuned candidate on the shared `rocke.dispatch.tuning` template (`rocke/dispatch/ARCHITECTURE.md` §14): its knob space holds the §4.10 knobs (axes in @@ -524,8 +522,9 @@ than refused, and the result's explanation names it (`dropped out-of-scope tunin value outside its knob's axis is refused. So `tuning_knobs` alone (`tuning_id` `auto`) carry a configuration across shapes. A stored `tuning_id` does not: it names the knob set that was canonical where it was recorded, and where scope drops one of those knobs the pin is -refused, naming the dropped knobs. `auto` runs the selected tile's base configuration: every -knob at its default (flags off, hints `default`, `waves_per_eu` `0`). +refused, naming the dropped knobs. A bare `spec_id` pin (no `tuning_id`, no knobs) runs the +tile's base configuration: every knob at its default (flags off, hints `default`, `waves_per_eu` +`0`). Not every listed configuration has run on a device. For one request the space is large (GDN, plain, bf16 state, D128: 54 tiles and 2,384 configurations at `production`). The fp32-checked @@ -535,39 +534,74 @@ combinations and `waves_per_eu` `1`–`3` and `5`–`8` are pinnable but not dev space is meant to shrink: a follow-up measures every configuration, each checked against the fp32 reference before it is timed, and removes the knobs and values that do not help. -KDA remains keyed on `work = batch × num_v_heads`. Tensor-parallel sharding -changes `num_v_heads` per rank, so two launches with the same batch can expose -different amounts of GPU work: +**`auto` defaults.** Production `auto` runs one static configuration per *case*: gate kind × +fused (either `fuse_conv` or `fuse_out_norm`) × state width. Each is a tile plus a knob stack on +top of that tile's base, kept as data in `dispatch/gdn/axes.py` (`AUTO_DEFAULTS`). Batch, head +count and `batch × num_v_heads` never select it: they change the grid only. Every case turns on +streaming (nontemporal) state loads and stores; the table lists the rest. A knob not listed stays +off (`waves_per_eu` `0`). -| `spec_id` | Work | `(num_warps, warp_threads_k, blocks_per_v_dim)` | -| --- | --- | --- | -| `kda_nw4_wtk16_bpv4` | `≤ 128` | `(4, 16, 4)` | -| `kda_nw1_wtk16_bpv4` | `≤ 512` | `(1, 16, 4)` | -| `kda_nw2_wtk16_bpv1` | larger | `(2, 16, 1)` | - -`BPV` manufactures workgroups when the natural grid is too small. KDA's table -comes from exhaustive legal-tile sweeps with every candidate correctness-gated -before timing. Its band edges interpolate measured anchors; exact measurements -live in the protected performance record. Both this table and the GDN default -were measured on the interleaved load order, before §4.3's loads-first order, -so they are a snapshot until the sweep is re-run. - -The request also carries the state width and the fused modes (§4.9). An `f32` state -(`state_dtype` `f32`, for either gate kind) reaches the spec and the kernel name but not the tile -choice: `auto` picks what it picks for the same request with a 16-bit state. Fused requests -(`fuse_conv` and/or `fuse_out_norm`) admit only `BPV = 1` tiles. Without a fused default, `auto` -would serve a fused GDN request with the first legal tile in product order, `(1, 1, 1)`, because -GDN's default `(2, 16, 8)` is not `BPV = 1`, and every fused KDA request with its largest-work -tile, `(2, 16, 1)`. `auto` therefore prefers one interim static fused tile, `(4, 16, 1)`, for both -gate kinds and every state width (`kda_nw4_wtk16_bpv1` for KDA). It is the tile the fused numeric -suite is built around, not a measured winner; per-mode fused defaults are a separate measurement. -It needs `DK` a multiple of `WTK × VPT = 128` and `DV` a multiple of 16, which both §1.4 -deployments meet. If it is illegal, `auto` falls back to the first legal candidate; for example, -fused GDN at `DK = 64` falls back to `(1, 1, 1)`, and at `DV = 8` to `(1, 8, 1)`. KDA's work table -never selects it for an unfused request: KDA's fallback order (taken only when the auto tile is -illegal) lists the work-table tiles, then this tile, then the GDN order. The fused flags must be -real bools and the dims real ints in the request, and `fuse_conv` with `num_k_heads ≠ num_v_heads` -is a request error (§4.9). +| Gate | Mode | State | Tile `(NW, WTK, BPV)` | Knobs besides the streaming state hints | +| --- | --- | --- | --- | --- | +| GDN | plain | bf16 / f16 | `(2, 16, 8)` | `dpp_reduce` | +| GDN | plain | f32 | `(2, 16, 16)` | `dpp_reduce`, `interleave_cols` | +| GDN | fused | bf16 / f16 | `(4, 16, 1)` | `dpp_reduce`, `stream_rows`, `conv_once`, `norm_gate_once`, `out_lds` | +| GDN | fused | f32 | `(8, 16, 1)` | `dpp_reduce`, `stream_rows`, `interleave_cols`, `conv_once`, `out_lds`, `waves_per_eu=4` | +| KDA | plain | bf16 / f16 | `(1, 16, 8)` | `dpp_reduce`, `waves_per_eu=4` | +| KDA | plain | f32 | `(1, 16, 16)` | `dpp_reduce`, `stream_rows`, `interleave_cols` | +| KDA | fused | bf16 / f16 | `(4, 8, 1)` | `dpp_reduce`, `conv_once`, `out_lds` | +| KDA | fused | f32 | `(4, 8, 1)` | `dpp_reduce`, `stream_rows`, `interleave_cols`, `conv_once`, `out_lds` | + +Provenance. The table comes from a cold-memory sweep on gfx950 (ROCm 7.13 container, Python +lowerer; every call reads its state from HBM; ≥ 1 GiB of inputs rotate per cycle) with bf16 I/O +and `use_qk_l2norm` on. A *shape* is one head geometry `Hk/Hv` at one batch. Each case was +measured at `DK = DV = 128` and batch 1, 8, 16, 32, 64, 128 and 256 on three head geometries: +KDA 8/8, 32/32, 96/96; GDN plain 16/32, 4/8, 32/32; +GDN fused 8/8, 16/16, 32/32. Of these, 16/32 is Qwen3-Next and 32/32 is Kimi-Linear (§1.4), 4/8 +and 8/8 are their four-way tensor-parallel splits, and the rest are validation points rather than +deployments. No §1.4 GDN deployment has `Hk = Hv`, so the GDN fused rows have no deployment +anchor. The fused cases were measured with both `fuse_conv` and `fuse_out_norm` on (`Hk = Hv`). +Stage 1 timed every legal tile × knob configuration of each case, correctness-gated against the +fp32 reference; stage 2 retimed the leaders (the stage-1 leaders and the best knob set on the +next tiles), arms interleaved in one process, three independent runs, every arm fp32-gated in the +measured run. + +Rule. A configuration's score is the geomean, over the shapes, of its time over the shape's +fastest configuration. A case takes the lowest score; among the configurations near-tied with it, +the lower worst-shape ratio, near-tied worst shapes counting as tied; then fewer non-default +knobs; then the lower score. One override, chosen for batch 1 (the latency case): for KDA fused +f32 the rule picks `(16, 16, 1)` with `interleave_cols`, `conv_once`, `out_lds`, +`waves_per_eu=4`; the row shipped is the `(4, 8, 1)` configuration with the best score among the +`(4, 8, 1)` arms, near-tied with the rule's pick on the score. One row serves every batch. + +Not measured: a fused request with one flag (it runs its row with the other flag's knobs +dropped, below; e.g. `fuse_out_norm` alone at `Hk ≠ Hv`, where `fuse_conv` is not allowed), other +head dims where the case tile is legal (e.g. GDN at `DK = 256`, which runs the row; with +`waves_per_eu=4` the GDN fused f32 row spills there), an `f16` state (it moves the same bytes as +bf16, so it runs the bf16 row), f16 I/O and `use_qk_l2norm` off (each runs its row), and gfx942. +To revisit a case, sweep its tiles × knobs cold (`tune.py` times tiles at their base +configuration only; an in-tree tiles × knobs sweep is a follow-up, §8), then change its row in +`AUTO_DEFAULTS`, the literal table in `test_gfx950_wiring.py` and this table together. + +A knob the request cannot read is dropped for it (the scope rule above): a request with only one +fused flag drops the knobs of the other (`out_lds` and `norm_gate_once` need `fuse_out_norm`; +`conv_once` needs `fuse_conv`). `norm_gate_once` also needs `conv_once` until §8 follow-up 8, so +a case row that carries it drops it wherever `conv_once` is dropped. The dispatched configuration +is a registered one: its result carries `spec_id`, `tuning_id` and `tuning_knobs`, which replay +it. + +Fused requests (§4.9) admit only `BPV = 1` tiles, which every fused row has. The fused flags must +be real bools and the dims real ints in the request, and `fuse_conv` with +`num_k_heads ≠ num_v_heads` is a request error (§4.9). + +**Fallback.** Each gate kind's priority order is its auto tiles (table order), then every other +tile in product order. When a case's tile is illegal for a request, `auto` runs the first legal +tile in that order at its base configuration (every knob off). Which tile that is depends on +`DK` and `DV`: a GDN request at `DK = DV = 64` or `192`, which cannot split over 16 K lanes +(`WTK × VPT = 128`), runs `(1, 1, 1)`, which is register-heavy (known limitation, §8); a plain GDN +request at `DK = 128` with `DV = 16` or `32` runs another case's `(4, 16, 1)`. A knob stack the +tile's knob space refuses for a request on which that tile is legal is a data error: dispatch +raises with the reason rather than run another configuration. ### 4.7 The reference path @@ -670,7 +704,7 @@ hints and `waves_per_eu` also apply on the reference path). Each one's default i kernel emitted before the knob existed, so with every knob at its default the kernel name and the emitted IR are byte-identical to the knob-free emitter (the golden IR test pins this). The registry exposes them as each tile's tuning space (§4.6), so a knob is reached by pinning a -configuration; `auto` keeps every knob at its default until per-mode defaults are measured. +configuration. `auto` runs one knob stack per case (§4.6). | Knob | Values (default first) | Changes code where | Effect | | --- | --- | --- | --- | @@ -954,7 +988,8 @@ Widening the range needs nested chunking or per-token rescaling. **Follow-ups.** 1. A fused-path GDN prefill kernel (§5.4). -2. gfx942 support; KDA work bands are arch-specific and need re-sweeping. +2. gfx942 support; the per-case `auto` defaults (§4.6) are gfx950 measurements and need + re-sweeping there. 3. Extending the supported decay range. 4. Scan-side parallelism beyond the current `value_splits` cap, or a shorter serial chain — the scan is the critical path at small `BH` (§5.4). @@ -966,12 +1001,16 @@ Widening the range needs nested chunking or per-token rescaling. `rocke.dispatch.tuning` template) is Python-only, with no C++ twin to keep in parity: the provider does not serve GDN/KDA decode through a C++ selector today. A C++ selector, if one is added, needs the same request, scope and pin rules and a parity gate. -7. A fast fused decode path (§4.9): lifting the three `NOT_YET_IMPLEMENTED` rules of §4.9 — - `fuse_conv` with `Hk ≠ Hv` (GQA, e.g. Qwen3-Next) and `BPV > 1` for small-batch parallelism, - through an out-of-place conv state, with `fuse_out_norm` at `BPV > 1` also needing a - cross-workgroup norm; and the fusions on the `simple` reference path. Computing each conv - channel and gate once and sharing it through LDS is now the `conv_once` / `norm_gate_once` knobs - (§4.10); choosing when to use them is still open. +7. A fast fused decode path (§4.9): an out-of-place conv state, which lifts two of the three + `NOT_YET_IMPLEMENTED` rules of §4.9 — `fuse_conv` with `Hk ≠ Hv` (GQA, e.g. Qwen3-Next) and + `BPV > 1` (with a cross-workgroup norm) for small-batch parallelism; and the fusions on the + `simple` reference path, the third. Computing each conv channel and gate once and sharing it + through LDS is now the `conv_once` / `norm_gate_once` knobs (§4.10); the fused `auto` defaults + (§4.6) say which each case runs. 8. `norm_gate_once` without `conv_once`: its `act(out_gate) ⊙ norm_weight` products need their own LDS block and barrier, so a norm-only spec can use it; rejected as `NOT_YET_IMPLEMENTED` until then (§4.8). +9. A footprint-based `auto` fallback order (§4.6): when a case's tile is illegal, the first legal + tile in priority order can be register-heavy (`(1, 1, 1)` for GDN at `DK = DV = 64` or `192`). +10. An in-tree tiles × knobs sweep (§4.6) that regenerates `AUTO_DEFAULTS`: `tune.py` times tiles + at their base configuration only. diff --git a/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/gdn/CASE_STUDY.md b/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/gdn/CASE_STUDY.md index 98797176b8a4..527669b8ace7 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/gdn/CASE_STUDY.md +++ b/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/gdn/CASE_STUDY.md @@ -49,21 +49,21 @@ Tiles are `(num_warps, warp_threads_k, blocks_per_v_dim)`. bf16 I/O, bf16 and - **Dispatcher tiles (114 tile and state-dtype pairs):** GDN at `Hk/Hv = 16/32`, every tile in [`CONFIGURED_TILES`](../../../dispatch/gdn/gfx950.py); KDA at - `32/32`, the three tiles in - [`_TUNED_TILES_KDA`](../../../dispatch/gdn/gfx950.py); each with bf16 and - `f32` state. This table was collected before dispatch accepted `f32` state, - so the `f32` half was built as specs directly. Dispatch can now request an - `f32` state but still uses the same auto tile as the 16-bit state. + `32/32`, the three tiles of the work-keyed table KDA `auto` used then, + `(4,16,4)`, `(1,16,4)` and `(2,16,1)`; each with bf16 and `f32` state. This + table was collected before dispatch accepted `f32` state, so the `f32` half + was built as specs directly. `auto` now runs one tile and knob stack per case + ([ALGORITHM.md §4.6](ALGORITHM.md#46-registry-and-tile-selection)). - **Auto-route fallback (4):** GDN `(1,1,1)` at `DK = DV = 64` and `192`, where - [`DEFAULT_TILE`](../../../dispatch/gdn/gfx950.py) `(2,16,8)` is illegal + the plain 16-bit `auto` tile `(2,16,8)` is illegal ([ALGORITHM.md §4.6](ALGORITHM.md#46-registry-and-tile-selection)). - **Fused tiles (228):** GDN and KDA at `16/16`, `BPV = 1`, every legal `(NW, WTK)` with `NW` in 1–16 and `WTK` in 1–32, with `fuse_conv`, `fuse_out_norm` (`rn` below) or both (`cv+rn`). This table was collected before dispatch had fusion request fields, so the measurements used - hand-built specs. Dispatch can now request fusion; `auto` routes the interim - `(4,16,1)` fused tile, and the other registered legal `BPV = 1` tiles remain - selectable by explicit `spec_id`. + hand-built specs. `auto` then routed one interim `(4,16,1)` fused tile; it + now runs each fused case's row (§4.6), and every other registered legal + `BPV = 1` tile stays selectable by explicit `spec_id`. The merge base is `153bf493555`, the interleaved order; the head is the load-first emitter. @@ -140,9 +140,9 @@ Fused tiles (228 cases; modeled waves per SIMD, merge base → head): `(4,16,1)` bf16 gains a wave on ROCm 7.1, loses one on ROCm 7.13 and does not change on ROCm 10). - **The exemption stays keyed on `NW == 1`.** The tiles that start to spill are - multi-wave `fuse_conv` tiles outside the interim `(4,16,1)` fused default. - Which ones spill differs per compiler, so a footprint-keyed exemption would - encode one compiler's register allocator. + multi-wave `fuse_conv` tiles outside the fused tiles `auto` runs (`(4,16,1)`, + `(8,16,1)` and `(4,8,1)`, §4.6). Which ones spill differs per compiler, so a + footprint-keyed exemption would encode one compiler's register allocator. - **The scratch test holds the stable subset.** [`test_lowers_and_compiles_without_scratch`](../../../tests/test_gdn_decode_fused.py) lowers all 228 fused cases and @@ -150,8 +150,8 @@ Fused tiles (228 cases; modeled waves per SIMD, merge base → head): every mode on `(1,16,1)`, `(2,8,1)`, `(2,16,1)`, `(4,8,1)`, `(4,16,1)` and `(8,16,1)`, plus norm-only on every other legal tile except `(1,1,1)` and `(1,2,1)`. -- **Tile defaults were picked on the interleaved order.** Re-run - [`tune.py`](tune.py) before relying on their ranking +- **The tiles timed in §6 were picked on the interleaved order.** The `auto` + defaults that replaced them were measured on the load-first order ([ALGORITHM.md §4.6](ALGORITHM.md#46-registry-and-tile-selection)). ## 6. Measured speed @@ -162,12 +162,11 @@ timings is not in this tree, so this section states the scope, method, correctness gate and result in words and publishes no speed numbers. **Scope.** Single-token `DK = DV = 128` decode, bf16 I/O, batches 1, 16 and 128. -GDN uses the default `(2,16,8)` tile. KDA uses the dispatcher's `(4,16,4)`, -`(1,16,4)` and `(2,16,1)` tiles; the fused f32 case uses `(4,16,1)`. The f32-state -and fused arms were built as specs directly; a dispatch request can now ask for -both, and `auto` serves a fused request `(4,16,1)`. No f32 or fused tile was -tuned. GDN fused performance was not measured. The tile choices were made on the -old load order ([§5](#5-decisions)). +GDN uses the `(2,16,8)` tile, its default at the time. KDA uses the `(4,16,4)`, +`(1,16,4)` and `(2,16,1)` tiles of the work-keyed table it used then; the fused +f32 case uses `(4,16,1)`. The f32-state and fused arms were built as specs +directly. No f32 or fused tile was tuned. GDN fused performance was not +measured. The tile choices were made on the old load order ([§5](#5-decisions)). **Method.** Both trees run in one process, with their arms interleaved, and each result is the median of 3 independent runs. State, conv-state and output @@ -201,12 +200,14 @@ from itertools import product from multiprocessing import Pool from pathlib import Path -from dispatch.gdn.gfx950 import _TUNED_TILES_KDA, CONFIGURED_TILES +from dispatch.gdn.gfx950 import CONFIGURED_TILES from kernels.gfx950.gdn_decode import GdnDecodeSpec, build_gdn_decode, is_valid_spec from rocke.analysis.isa import analyze_hsaco from rocke.helpers.compile import compile_kernel FUSED = ((True, False), (False, True), (True, True)) # (fuse_conv, fuse_out_norm) +# The KDA work-keyed tiles `auto` used when this table was collected. +KDA_TILES = ((4, 16, 4), (1, 16, 4), (2, 16, 1)) def specs(): @@ -214,7 +215,7 @@ def specs(): for nw, wtk, bpv in CONFIGURED_TILES: yield dict(gate_kind="gdn", num_k_heads=16, num_v_heads=32, state_dtype=st, num_warps=nw, warp_threads_k=wtk, blocks_per_v_dim=bpv) - for _, (nw, wtk, bpv) in _TUNED_TILES_KDA: + for nw, wtk, bpv in KDA_TILES: yield dict(gate_kind="kda", num_k_heads=32, num_v_heads=32, state_dtype=st, num_warps=nw, warp_threads_k=wtk, blocks_per_v_dim=bpv) for d in (64, 192): # (2,16,8) is illegal here, so auto takes (1,1,1) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/gdn/README.md b/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/gdn/README.md index a28bb47d59f3..e9ffd5e93064 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/gdn/README.md +++ b/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/gdn/README.md @@ -32,12 +32,12 @@ retune either decode mode or the GDN prefill path. | File | Purpose | |---|---| | [`gdn_decode.py`](gdn_decode.py) | Compile a spec, build inputs, launch the shared decode emitter, and compare with the independent fp32 reference | -| [`tune.py`](tune.py) | Measure legal GDN registry candidates or KDA work-keyed candidates | +| [`tune.py`](tune.py) | Measure every legal GDN registry tile or KDA tile (base configuration) against the `auto` tile | | [`ALGORITHM.md`](ALGORITHM.md) | Explain the gated delta rule, gate kinds, and GPU mapping | | [`CASE_STUDY.md`](CASE_STUDY.md) | Per-tile register, wave and scratch changes of the decode load-first order, per compiler, and the command that regenerates them | -| [`library/benchmarks/gfx950/gdn/benchmark_gdn_decode.py`](../../../benchmarks/gfx950/gdn/benchmark_gdn_decode.py) | Benchmark every legal GDN registry tile (base configuration) and the static dispatcher default | +| [`library/benchmarks/gfx950/gdn/benchmark_gdn_decode.py`](../../../benchmarks/gfx950/gdn/benchmark_gdn_decode.py) | Benchmark every legal GDN registry tile (base configuration) and the dispatcher `auto` configuration (its case's tile and knob stack) | | [`library/benchmarks/gfx950/gdn/benchmark_kda_decode.py`](../../../benchmarks/gfx950/gdn/benchmark_kda_decode.py) | Benchmark KDA fused/precomputed/simple variants from the dispatcher; optionally sweep all legal tiles | -| [`library/dispatch/gdn/gfx950.py`](../../../dispatch/gdn/gfx950.py) | Register every (gate kind, tile); declare the GDN static default, KDA work-keyed table, and the interim fused default tile | +| [`library/dispatch/gdn/gfx950.py`](../../../dispatch/gdn/gfx950.py) | Register every (gate kind, tile) and select the `auto` configuration of each case; the per-case defaults are data in [`axes.py`](../../../dispatch/gdn/axes.py) (`AUTO_DEFAULTS`) | | [`library/dispatch/gdn/axes.py`](../../../dispatch/gdn/axes.py), [`rules.py`](../../../dispatch/gdn/rules.py) | Each tile's knob space: the knob axes as data, and their scope and legality | | [`library/tests/dispatch/gdn/test_gfx950_registry.py`](../../../tests/dispatch/gdn/test_gfx950_registry.py) | CPU GDN registry count, identity, legality, and selection coverage | | [`library/tests/dispatch/gdn/test_tuning_contract.py`](../../../tests/dispatch/gdn/test_tuning_contract.py) | CPU tuning contract: knob space per mode, pins (`tuning_id` / `tuning_knobs`), refusals | @@ -150,8 +150,8 @@ assert max(out_err, state_err) < TOL ``` The same modes are reachable through dispatch. `GdnDecodeRequest` takes -`state_dtype="f32"` and the `fuse_conv` / `fuse_out_norm` flags; a fused -request's `auto` tile is an interim static default +`state_dtype="f32"` and the `fuse_conv` / `fuse_out_norm` flags; `auto` runs +one measured tile and knob stack per case, gate kind x fused x state width ([`ALGORITHM.md` §4.6](ALGORITHM.md#46-registry-and-tile-selection)). Launch the result's kernel spec with the builder's inputs, as for any dispatched spec: @@ -175,13 +175,14 @@ computes (the two state hints and `waves_per_eu` also apply on the reference path): `state_load_hint`, `state_store_hint`, `dpp_reduce`, `xcd_remap`, `stream_rows`, `interleave_cols`, `conv_once`, `norm_gate_once`, `out_lds` and `waves_per_eu`. Every one defaults to the code the kernel emitted before it -existed, so a default spec keeps its name and its IR, and `auto` keeps every -knob at its default. Each registered tile's knob space holds them: every -(tile x knob) configuration is listed by `dispatch_gdn_decode_all` (and each -tile's base configuration by `dispatch_gdn_decode_bases`) and pinnable by its -`spec_id` plus `tuning_knobs` and/or `tuning_id`. The sweep levels and pin -rules are in [`ALGORITHM.md` §4.6](ALGORITHM.md#46-registry-and-tile-selection); -each knob's rules and the tiles where it changes code are in +existed, so a default spec keeps its name and its IR. `auto` runs its case's +knob stack (§4.6); a bare `spec_id` pin runs the tile with every knob at its +default. Each registered tile's knob space holds them: every (tile x knob) +configuration is listed by `dispatch_gdn_decode_all` (and each tile's base +configuration by `dispatch_gdn_decode_bases`) and pinnable by its `spec_id` +plus `tuning_knobs` and/or `tuning_id`. The sweep levels and pin rules are in +[`ALGORITHM.md` §4.6](ALGORITHM.md#46-registry-and-tile-selection); each knob's +rules and the tiles where it changes code are in [§4.10](ALGORITHM.md#410-performance-knobs). ```python @@ -196,8 +197,8 @@ result.spec.tuning_id # "nw2_wtk16_bpv8_wpe0@", stored with result. The GDN benchmark asks the registry for every legal tile and times its base configuration (`dispatch_gdn_decode_bases`: every knob at its default). The default D128 request admits 54 of 180 -configured triples; the benchmark also reports the static dispatcher-default -`auto` selection: +configured triples; the benchmark also reports the dispatcher `auto` +configuration: ```bash python3 library/benchmarks/gfx950/gdn/benchmark_gdn_decode.py \ @@ -206,9 +207,10 @@ python3 library/benchmarks/gfx950/gdn/benchmark_gdn_decode.py \ Each row includes a stable `spec_id` `nw_wtk_bpv`. Dispatcher `auto` -prefers `(2, 16, 8)` whenever legal; measurements never change that choice. +runs its case's static default whenever its tile is legal; measurements never +change that choice. -KDA remains a separate work-keyed benchmark: +KDA has its own benchmark of the dispatched kernel and its variants: ```bash python3 -m benchmarks.gfx950.gdn.benchmark_kda_decode \ @@ -231,11 +233,13 @@ python3 library/builders/gfx950/gdn/tune.py \ --gate-kind gdn --batches 1,16,64,256 --top 5 ``` -For every GDN cell, it also reports the static dispatcher default, the fastest -legal candidate, the default's rank and time ratio, and a manual -`DEFAULT_TILE` recommendation. Measurements never update the shipped default. +For every GDN cell, it also reports the `auto` tile at its base configuration, +the fastest legal candidate, that tile's rank and time ratio, and a note that +base-configuration timings are no basis for changing `AUTO_DEFAULTS`. Both gate +kinds time tiles at their base configuration only (every knob off), not the +`auto` knob stack; measurements never update the shipped default. -Use KDA's work-keyed study across head geometries when retuning KDA: +The KDA tile study across head geometries: ```bash python3 library/builders/gfx950/gdn/tune.py \ @@ -243,8 +247,12 @@ python3 library/builders/gfx950/gdn/tune.py \ --batches 1,2,4,8,16,32,64,128 --top 5 ``` -GDN measurements are report-only. KDA's measured winners may justify updating -its work-keyed bands; rerun dispatch wiring and numeric tests after doing so. +Both are report-only. Changing an `auto` default needs a cold-memory sweep of +the case's tiles x knobs, which `tune.py` does not do (an in-tree sweep is a +follow-up, [`ALGORITHM.md` §8](ALGORITHM.md#8-known-limits-and-follow-ups)). Change the row in +`AUTO_DEFAULTS`, the literal table in `test_gfx950_wiring.py` and the table in +[`ALGORITHM.md` §4.6](ALGORITHM.md#46-registry-and-tile-selection) together, +then rerun the dispatch wiring, golden and numeric tests. ## Run tests @@ -343,9 +351,9 @@ For the simple reference path, there is no V split: grid = batch * num_v_heads ``` -For GDN, `auto` always uses the static `(2, 16, 8)` registry priority when it -is legal; batch changes the grid but not the tile. KDA alone changes its -work-keyed table choice with `batch * num_v_heads`. +`auto` runs one static tile and knob stack per case (gate kind x fused x state +width) whenever the tile is legal; batch and `batch * num_v_heads` change the +grid but not the configuration. `out_err` and `state_err` are maximum absolute errors against the fp32 reference (except `out_err` of a `fuse_out_norm` spec, relative to `max(1, |ref|)`; see diff --git a/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/gdn/tune.py b/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/gdn/tune.py index b306a816a232..eddd6edc2d6b 100755 --- a/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/gdn/tune.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/gdn/tune.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 # Copyright (c) Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT -"""Measure gfx950 GDN registry candidates and KDA tile-table alternatives. +"""Measure the gfx950 GDN/KDA decode tiles against the ``auto`` tile. -GDN dispatch has a static registry priority. Its sweep measures every legal -registered candidate but does not change dispatcher-default selection. KDA keeps -its separate work-keyed table, so its sweep enumerates every validator-admitted -tile to challenge the selected work band. +``auto`` is one static tile and knob stack per case (``dispatch.gdn.axes. +AUTO_DEFAULTS``). This tool times tiles only, each at its base configuration +(every knob off), and marks the tile ``auto`` dispatches; it does not time the +knob stack and never changes selection. GDN measures every legal registered +candidate; KDA enumerates every validator-admitted tile. A tiles x knobs sweep +is not in this tool. Every candidate is correctness-gated before device timing. Host launch cost can hide kernel differences at small batch, so device time is the comparison metric. @@ -15,7 +17,7 @@ PYTHONPATH=/library:/platform/python python3 tune.py -Run the KDA work-keying study across several head geometries:: +Run the KDA tile study across several head geometries:: PYTHONPATH=/library:/platform/python python3 tune.py --gate-kind kda \ --geometries 16/32,8/16,4/8 \ @@ -34,7 +36,7 @@ dispatch_gdn_decode, dispatch_gdn_decode_bases, ) -from dispatch.gdn.gfx950 import BLOCKS_PER_V_DIM, NUM_WARPS, WARP_THREADS_K, work_for +from dispatch.gdn.gfx950 import BLOCKS_PER_V_DIM, NUM_WARPS, WARP_THREADS_K, make_spec from kernels.gfx950.gdn_decode import GdnDecodeSpec, is_valid_spec ARCH = "gfx950" @@ -223,62 +225,38 @@ def sweep_batch(base: GdnDecodeSpec, batch: int, configs): def report_gdn_dispatcher_default(rows, auto_id: str) -> bool: - """Print the shipped GDN default against the fastest measured candidate. + """Print the ``auto`` tile's base configuration against the fastest + measured candidate (``auto`` itself also runs its case's knob stack). - Return False when the default is not among the correct, timed rows. + Return False when the auto tile is not among the correct, timed rows. """ best_micros, best_tile, best_id, _ = rows[0] for rank, (micros, tile, spec_id, _) in enumerate(rows, start=1): if spec_id != auto_id: continue print( - f" dispatcher default: {micros:.3f}us {spec_id} " + f" auto tile (base configuration): {micros:.3f}us {spec_id} " f"tile={tile} rank={rank}/{len(rows)}" ) print( f" fastest legal candidate: {best_micros:.3f}us {best_id} " f"tile={best_tile}" ) - print(f" default / fastest = {micros / best_micros:.3f}x") - if spec_id == best_id: - print(" manual review: retain DEFAULT_TILE") - else: - print(f" manual review: consider DEFAULT_TILE = {best_tile}") + print(f" auto tile base / fastest = {micros / best_micros:.3f}x") + # Base-configuration timings cannot re-pick a row: auto runs its + # case's tile with a knob stack, chosen on tiles x knobs. + print( + " base configurations only; not a basis for changing AUTO_DEFAULTS " + "(ALGORITHM.md section 4.6)" + ) return True print( - f" dispatcher default {auto_id!r} is NOT in the correct-and-timeable " + f" auto tile {auto_id!r} is NOT in the correct-and-timeable " "set for this cell" ) return False -def report_kda_work_keying(by_work) -> None: - """Check that cells sharing ``batch * num_v_heads`` pick the same best tile. - - ``_TUNED_TILES_KDA`` is keyed on work alone. Cells with equal work but a - different (batch, num_v_heads) split are the evidence for or against that - key: a disagreement invalidates the table's key, not just one value. - """ - print("\n=== KDA work -> best tile, across geometries ===") - print(f"{'work':>7} {'best tile':16} {'us':>9} cells (batch x Hv)") - disagreements = 0 - for work in sorted(by_work): - cells = by_work[work] - tiles = {cell[1] for cell in cells} - fastest = min(cells) - cell_text = " ".join(f"{batch}x{hv}" for _, _, batch, hv in cells) - flag = "" if len(tiles) == 1 else " <-- TILES DISAGREE" - disagreements += len(tiles) > 1 - print(f"{work:>7} {str(fastest[1]):16} {fastest[0]:9.3f} {cell_text}{flag}") - if disagreements: - print( - f"\nWARNING: work alone did not fix the best tile at {disagreements} " - "work value(s). Check whether the disagreeing times sit inside " - "run-to-run variation; if they do not, _TUNED_TILES_KDA must not be " - "keyed on work." - ) - - def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -310,7 +288,6 @@ def main() -> int: for item in args.geometries.split(",") ] failed = False - by_work = {} # KDA work -> [(us, best tile, batch, num_v_heads), ...] for num_k_heads, num_v_heads in geometries: for batch in batches: request = GdnDecodeRequest( @@ -321,11 +298,17 @@ def main() -> int: num_v_heads=num_v_heads, ) if args.gate_kind == "kda": - # KDA's measured table is work-keyed, but the study must test - # every validator-admitted tile rather than the current band's - # dispatcher result. - auto = dispatch_gdn_decode(request) - base = auto.spec.kernel_spec + # Every validator-admitted tile, not only the registered + # candidates' order, each at its base configuration: the + # sweep starts from the dispatched tile with every knob off, + # not from ``auto``'s knob stack. + kernel = dispatch_gdn_decode(request).spec.kernel_spec + auto_tile = ( + kernel.num_warps, + kernel.warp_threads_k, + kernel.blocks_per_v_dim, + ) + base = make_spec(request, auto_tile) configs = legal_configs(base) print(f"legal KDA configurations for batch {batch}: {len(configs)}") rows = sweep_batch(base, batch, configs) @@ -333,26 +316,18 @@ def main() -> int: print(f"batch {batch}: no candidate was both correct and timeable") failed = True continue - auto_tile = ( - base.num_warps, - base.warp_threads_k, - base.blocks_per_v_dim, - ) print( f"\n=== Hk{num_k_heads}/Hv{num_v_heads} batch {batch}: " f"top {args.top} ===" ) for micros, tile, err in rows[: args.top]: - mark = " <- dispatcher default" if tile == auto_tile else "" + mark = " <- auto tile" if tile == auto_tile else "" print(f" {micros:9.3f}us tile={tile} err={err:.2e}{mark}") if all(tile != auto_tile for _, tile, _ in rows): print( - f" dispatcher default {auto_tile} is NOT in the " + f" auto tile {auto_tile} is NOT in the " "correct-and-timeable set for this cell" ) - by_work.setdefault(work_for(batch, num_v_heads), []).append( - (rows[0][0], rows[0][1], batch, num_v_heads) - ) continue # Each registered tile's base configuration. @@ -370,15 +345,14 @@ def main() -> int: f"top {args.top} ===" ) for micros, tile, spec_id, err in rows[: args.top]: - mark = " <- dispatcher default" if spec_id == auto_id else "" + mark = " <- auto tile" if spec_id == auto_id else "" print(f" {micros:9.3f}us {spec_id} tile={tile} err={err:.2e}{mark}") if not report_gdn_dispatcher_default(rows, auto_id): failed = True - if by_work: - report_kda_work_keying(by_work) print( - "\nDispatcher default is deterministic; measurements do not change selection." + "\nThe auto configuration is deterministic; measurements do not change " + "selection." ) return 1 if failed else 0 diff --git a/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/__init__.py b/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/__init__.py index 805649b4dfd2..7841ad616521 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/__init__.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/__init__.py @@ -78,11 +78,10 @@ def dispatch_gdn_decode_all(req: GdnDecodeRequest) -> Tuple[DispatchResult, ...] ``axes.py``). Tiles come in registry priority order, and each tile's base configuration - (every knob at its default, what ``auto`` runs on that tile) comes first - among its configurations. Each result's request is pinned to its spec - (``spec_id``, ``tuning_id``, ``tuning_knobs``), so dispatching it again - selects exactly that kernel. :func:`dispatch_gdn_decode_bases` keeps only - the base configurations. + (every knob at its default) comes first among its configurations. Each + result's request is pinned to its spec (``spec_id``, ``tuning_id``, + ``tuning_knobs``), so dispatching it again selects exactly that kernel. + :func:`dispatch_gdn_decode_bases` keeps only the base configurations. """ if request_errors(req): return () @@ -90,8 +89,8 @@ def dispatch_gdn_decode_all(req: GdnDecodeRequest) -> Tuple[DispatchResult, ...] def dispatch_gdn_decode_bases(req: GdnDecodeRequest) -> Tuple[DispatchResult, ...]: - """Each legal registered tile's base configuration (no knobs: what ``auto`` - runs on that tile), one result per tile in registry priority order. What a + """Each legal registered tile's base configuration (no knobs: what a bare + ``spec_id`` pin runs), one result per tile in registry priority order. What a caller that compares tiles wants, rather than every configuration :func:`dispatch_gdn_decode_all` lists. @@ -142,14 +141,14 @@ def dispatch_gdn_decode( ) -> DispatchResult: """Select a registered GDN decode configuration for ``req``. - ``auto`` returns the static tile's base configuration (every knob at its - default). A ``spec_id`` pin (with optional ``tuning_id`` / - ``tuning_knobs``) returns exactly that configuration or raises - :class:`rocke.dispatch.core.PinRefused` with the candidate's reason; it - never falls back to another kernel. The result's request is pinned to the - selected spec, so it replays the same kernel; ``result.spec`` is a - :class:`GdnDecodeTunedSpec`, whose ``kernel_spec`` is what the builder and - launcher read. + ``auto`` returns the request's case default (``axes.AUTO_DEFAULTS``: one + tile plus a knob stack per gate kind x fused x state width). A + ``spec_id`` pin (with optional ``tuning_id`` / ``tuning_knobs``) returns + exactly that configuration or raises :class:`rocke.dispatch.core.PinRefused` + with the candidate's reason; it never falls back to another kernel. The + result's request is pinned to the selected spec, so it replays the same + kernel; ``result.spec`` is a :class:`GdnDecodeTunedSpec`, whose + ``kernel_spec`` is what the builder and launcher read. """ candidate = GDN_REGISTRY.select(req, ranker=ranker) spec = candidate.select_spec(req) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/axes.py b/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/axes.py index 40cfe19452bf..129ee6c14932 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/axes.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/axes.py @@ -1,7 +1,7 @@ # Copyright (c) Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT -"""The gfx950 GDN/KDA decode tuning space as data: the knob axes and the -``waves_per_eu`` values. +"""The gfx950 GDN/KDA decode tuning space as data: the knob axes, the +``waves_per_eu`` values, and the static ``auto`` default of every case. Nothing here walks or builds a spec: the axis helpers and the walk are the shared :mod:`rocke.dispatch.tuning`, and :mod:`.rules` decides which axes a @@ -9,10 +9,11 @@ Every knob is a :class:`~kernels.gfx950.gdn_decode.GdnDecodeSpec` field whose dataclass default is the code the kernel emitted before the knob existed, so -each registered tile's base configuration (no knobs) is the kernel ``auto`` -selected before the registry existed. What each knob does is documented on the -spec field (ALGORITHM.md §4.10); where it changes code is the kernel's -``*_applies`` predicates and ``is_valid_spec``: +each registered tile's base configuration (no knobs) is that code. ``auto`` +runs one ``AUTO_DEFAULTS`` entry per case: a tile plus knobs on top of its +base. What each knob does is documented on the spec field (ALGORITHM.md +§4.10); where it changes code is the kernel's ``*_applies`` predicates and +``is_valid_spec``: * ``state_load_hint`` / ``state_store_hint``: cache policy of the recurrent-state loads and stores (``"default"`` or ``"streaming"``); @@ -32,7 +33,8 @@ from __future__ import annotations import dataclasses as dc -from typing import Tuple +from types import MappingProxyType +from typing import Mapping, Tuple from kernels.gfx950.gdn_decode import _MAX_WAVES_PER_EU, STATE_HINTS, GdnDecodeSpec from rocke.dispatch.tuning.axes import KnobAxis, flag, values @@ -78,3 +80,114 @@ # is pinnable at either level; the level only decides what is listed. PRODUCTION_WAVES_PER_EU: Tuple[int, ...] = (0, 4) FULL_WAVES_PER_EU: Tuple[int, ...] = tuple(range(_MAX_WAVES_PER_EU + 1)) + + +@dc.dataclass(frozen=True) +class AutoDefault: + """One case's ``auto`` configuration: a registered tile and the knobs set + on top of that tile's base configuration (every other knob stays at its + dataclass default).""" + + tile: Tuple[int, int, int] + knobs: Mapping[str, object] + + +def _auto(tile: Tuple[int, int, int], **knobs) -> AutoDefault: + return AutoDefault(tile, MappingProxyType(dict(_STREAMING_STATE, **knobs))) + + +# Every case streams the state: nontemporal state loads and stores (the state +# is read once and written once per call). Each row lists its other knobs; +# a knob it does not list stays at its default (off, waves_per_eu 0). +_STREAMING_STATE = {"state_load_hint": "streaming", "state_store_hint": "streaming"} + +# The state width an ``auto`` request reads its row with. f16 state was not +# measured; it moves the same bytes as bf16, so it runs the bf16 row. +AUTO_STATE_ROW: Mapping[str, str] = MappingProxyType( + {"bf16": "bf16", "f16": "bf16", "f32": "f32"} +) + +# The static ``auto`` default per case: (gate_kind, fused, state row), where +# fused means fuse_conv or fuse_out_norm. One entry per case, never keyed on +# batch, head count or work: the batch only changes the grid. Chosen on gfx950 +# in cold memory at head_k_dim = head_v_dim = 128 over three head geometries +# per case and batch 1..256 (ALGORITHM.md §4.6 names the shapes; the fused +# cases were timed with both fuse_conv and fuse_out_norm): a screen of every +# legal tile x knob configuration, then a retime of the leaders (arms +# interleaved, three runs). Rule: the lowest geomean against each shape's +# fastest configuration; among the configurations near-tied with it, the lower +# worst-shape ratio, near-tied worst shapes counting as tied; then fewer +# non-default knobs; then the lower geomean. gfx942 is not measured. +# +# A knob a request cannot read is dropped for it, never refused (rules.py +# scope): the fused-only knobs on a request with only one fused flag (and +# norm_gate_once with conv_once, which the kernel requires for it), and +# dpp_reduce / stream_rows / interleave_cols where the tile's ``*_applies`` +# predicate is False. To revisit, sweep the case's tiles x knobs cold, then +# change the row here, the literal table in test_gfx950_wiring.py and the +# table in ALGORITHM.md §4.6 together. +AUTO_DEFAULTS: Mapping[Tuple[str, bool, str], AutoDefault] = MappingProxyType( + { + ("gdn", False, "bf16"): _auto((2, 16, 8), dpp_reduce=True), + ("gdn", False, "f32"): _auto( + (2, 16, 16), dpp_reduce=True, interleave_cols=True + ), + ("gdn", True, "bf16"): _auto( + (4, 16, 1), + dpp_reduce=True, + stream_rows=True, + conv_once=True, + norm_gate_once=True, + out_lds=True, + ), + ("gdn", True, "f32"): _auto( + (8, 16, 1), + dpp_reduce=True, + stream_rows=True, + interleave_cols=True, + conv_once=True, + out_lds=True, + waves_per_eu=4, + ), + ("kda", False, "bf16"): _auto((1, 16, 8), dpp_reduce=True, waves_per_eu=4), + ("kda", False, "f32"): _auto( + (1, 16, 16), dpp_reduce=True, stream_rows=True, interleave_cols=True + ), + ("kda", True, "bf16"): _auto( + (4, 8, 1), dpp_reduce=True, conv_once=True, out_lds=True + ), + # An override of the rule, chosen for batch 1 (the latency case): the + # rule picks (16, 16, 1) with interleave_cols, conv_once, out_lds and + # waves_per_eu 4; this row is the (4, 8, 1) configuration with the + # best selection score among the (4, 8, 1) arms, near-tied with the + # rule's pick on that score. + ("kda", True, "f32"): _auto( + (4, 8, 1), + dpp_reduce=True, + stream_rows=True, + interleave_cols=True, + conv_once=True, + out_lds=True, + ), + } +) + +# Each row's knobs are knobs of the space and its values values of their axes +# (waves_per_eu: an int the kernel accepts), so a mistyped row fails at +# import; a well-formed stack the space refuses on a request raises at +# dispatch (``auto_knobs``), never runs another configuration. +_AXIS_VALUES = { + axis.name: {value for choice in axis.choices for _, value in choice} + for axis in AXES +} +_AXIS_VALUES["waves_per_eu"] = set(FULL_WAVES_PER_EU) +for _case, _default in AUTO_DEFAULTS.items(): + for _name, _value in _default.knobs.items(): + _allowed = _AXIS_VALUES.get(_name, set()) + assert _value in _allowed and type(_value) in {type(v) for v in _allowed}, ( + _case, + _name, + _value, + ) + # Fused requests run one workgroup per head. + assert not _case[1] or _default.tile[2] == 1, _case diff --git a/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/common.py b/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/common.py index 1450ec58fe11..afefdacf19a4 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/common.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/common.py @@ -65,12 +65,12 @@ class GdnDecodeRequest(OperatorRequest): ``batch`` is the number of *active* sequences this step. It always contributes to the launch grid. - For ``gate_kind="gdn"``, ``auto`` uses the static ``DEFAULT_TILE`` whenever - it is legal; otherwise dispatch chooses a validator-admitted fallback. - Neither ``batch`` nor ``num_v_heads`` chooses GDN's auto tile. - - For ``gate_kind="kda"``, ``auto`` is keyed on ``batch * num_v_heads`` -- the - "work" -- so tensor parallelism selects the tile for heads local to a rank. + ``auto`` runs one static configuration per case -- gate kind x fused + (either flag) x state width, a tile plus a knob stack + (``dispatch.gdn.axes.AUTO_DEFAULTS``; an f16 state uses the bf16 row). + Neither ``batch`` nor the head counts choose it. When the case's tile is + illegal for the request, dispatch falls back to the first legal tile in + the gate kind's priority order, at its base configuration (every knob off). ``state_dtype`` may be ``"f32"`` as well as a 16-bit dtype, for both gate kinds; only the recurrent state widens, so ``dtype`` stays 16-bit. @@ -78,8 +78,8 @@ class GdnDecodeRequest(OperatorRequest): ``fuse_conv`` / ``fuse_out_norm`` select the optional fused modes: a causal conv1d + SiLU on the packed q/k/v row (in-place conv-state shift) and a gated RMSNorm on the output. Fused requests run one workgroup per - head, so only ``blocks_per_v_dim == 1`` tiles are legal, and ``auto`` uses - the static ``FUSED_DEFAULT_TILE`` for both gate kinds whenever it is legal. + head, so only ``blocks_per_v_dim == 1`` tiles are legal; every fused case's + ``auto`` tile has ``blocks_per_v_dim == 1``. ``fuse_conv`` also requires ``num_k_heads == num_v_heads``. Both flags change the kernel ABI, so they (and ``use_qk_l2norm``) must be real bools, and ``batch`` and the head counts and dims must be real ints: a request is @@ -105,10 +105,11 @@ class GdnDecodeRequest(OperatorRequest): configuration of that tile: ``tuning_knobs`` is the knob dict a sweep recorded next to the id (``{"dpp_reduce": True, ...}``). ``tuning_id`` ``"auto"`` with no knobs is the tile's base configuration (every knob - off), which is what ``auto`` runs. As for ``spec_id`` (the shared - dispatch convention), an empty or whitespace ``tuning_id`` means - ``"auto"``. A pin that does not resolve raises ``PinRefused``; it is - never mapped to another configuration. + off); ``auto``'s own configuration replays from the ``spec_id``, + ``tuning_id`` and ``tuning_knobs`` its result's request carries. As for + ``spec_id`` (the shared dispatch convention), an empty or whitespace + ``tuning_id`` means ``"auto"``. A pin that does not resolve raises + ``PinRefused``; it is never mapped to another configuration. """ batch: int diff --git a/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/gfx950.py b/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/gfx950.py index 4fdd8a295765..196347c226bc 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/gfx950.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/gfx950.py @@ -7,9 +7,9 @@ one tuned candidate per (gate kind, tile), whose knob space (``rules.py``, axes in ``axes.py``) holds every performance knob of that tile. The kernel validator remains the only authority that decides which configurations are -legal for a request. Production ``auto`` prefers one tile per request -(``auto_tile``: GDN's static default, KDA's work-keyed table, or the shared -fused default) with every knob at its default, and never consults runtime +legal for a request. Production ``auto`` runs one static configuration per +case (``auto_default``: gate kind x fused x state width, a tile plus a knob +stack, data in ``axes.AUTO_DEFAULTS``) and never consults runtime measurements. The tile is the variant, not an axis: each tile keeps its own ``spec_id``, @@ -28,12 +28,14 @@ from kernels.gfx950.gdn_decode import ( GDN_DTYPES, GdnDecodeSpec, + conv_once_applies, gdn_decode_signature, is_valid_spec, ) from rocke.dispatch.core import Capability, KernelCandidate, OperatorRequest from rocke.dispatch.tuning import make_tuned_candidate +from .axes import AUTO_DEFAULTS, AUTO_STATE_ROW, AutoDefault from .common import ( FAMILY, GDN_ABI_VERSION, @@ -45,62 +47,33 @@ ARCH = "gfx950" -# KDA keeps its measured work-keyed table. GDN candidate registration below -# owns the full configured tile space and has no GDN batch-winner table. -# Both this table and DEFAULT_TILE below were measured on the interleaved -# load order, before the warp-tiled emitter issued every load first -# (ALGORITHM.md §4.3); re-run tune.py before relying on their ranking. -# -# (max_work, (num_warps, warp_threads_k, blocks_per_v_dim)) -_TUNED_TILES_KDA = ( - (128, (4, 16, 4)), - (512, (1, 16, 4)), - (None, (2, 16, 1)), -) - NUM_WARPS = (1, 2, 4, 8, 16) WARP_THREADS_K = (1, 2, 4, 8, 16, 32) BLOCKS_PER_V_DIM = (1, 2, 4, 8, 16, 32) -# GDN ``auto`` is one static tile on purpose; batch only changes the grid. -# This replaced a batch-keyed table that picked (4,16,8) / (2,8,2) / (1,8,1) / -# (8,16,1) for batch <=4 / <=32 / <=128 / larger. No single legal tile matches -# all four of those winners. (2,16,8) was measured on gfx950 against that table -# at batch 1/16/64/256: batch 256 was neutral, batch 64 is slower, and batch 1 -# and 16 read slower but were too noisy to call. The batch-64 cost was accepted -# in exchange for one deterministic default. Across every legal tile at those -# batches, (2,16,8) has the lowest geomean and worst-case slowdown against each -# batch's fastest tile, though it is not the fastest at any single batch (on -# the interleaved load order; see above). To revisit, run -# ``tune.py --gate-kind gdn``; it reports this default's rank and its ratio to -# the fastest legal tile per batch. -DEFAULT_TILE = (2, 16, 8) - _LEXICOGRAPHIC_TILES = tuple(product(NUM_WARPS, WARP_THREADS_K, BLOCKS_PER_V_DIM)) -# Known limitation: when DEFAULT_TILE is illegal (e.g. head_k_dim 64/192), the -# fallback is the first legal tile in product order, (1,1,1), which is -# register-heavy. Needs a footprint-based fallback order. -CONFIGURED_TILES = (DEFAULT_TILE,) + tuple( - tile for tile in _LEXICOGRAPHIC_TILES if tile != DEFAULT_TILE -) -# Fused (fuse_conv / fuse_out_norm) requests need blocks_per_v_dim == 1, which -# GDN's DEFAULT_TILE is not. Interim static fused default for both gate kinds -# and every state width (ALGORITHM.md §4.6): the tile the fused numeric suite is -# built around, not a measured winner. It needs head_k_dim % 128 == 0 and -# head_v_dim % 16 == 0; otherwise (e.g. GDN head_k_dim 64) a fused request still -# falls back to (1,1,1), as for DEFAULT_TILE above. -FUSED_DEFAULT_TILE = (4, 16, 1) -assert FUSED_DEFAULT_TILE in CONFIGURED_TILES and FUSED_DEFAULT_TILE[2] == 1 - -# KDA registers the same configured tile space as GDN. Its fallback order -# (taken only when the request's auto tile is illegal) lists the work-table -# tiles in table order and the fused default first, then the GDN order. -_KDA_AUTO_TILES = tuple(tile for _, tile in _TUNED_TILES_KDA) + (FUSED_DEFAULT_TILE,) -assert all(tile in CONFIGURED_TILES for tile in _KDA_AUTO_TILES) -KDA_CONFIGURED_TILES = _KDA_AUTO_TILES + tuple( - tile for tile in CONFIGURED_TILES if tile not in _KDA_AUTO_TILES -) -CONFIGURED_TILES_BY_GATE = {"gdn": CONFIGURED_TILES, "kda": KDA_CONFIGURED_TILES} + +def _auto_tiles(gate_kind: str) -> Tuple[Tuple[int, int, int], ...]: + """The gate kind's ``AUTO_DEFAULTS`` tiles, in table order, once each.""" + tiles = (d.tile for (gate, _, _), d in AUTO_DEFAULTS.items() if gate == gate_kind) + return tuple(dict.fromkeys(tiles)) + + +# Each gate kind registers the whole tile space. Its priority order is its auto +# tiles first, then every other tile in product order. ``auto`` falls back to +# that order only when the case's tile is illegal for the request: the first +# legal tile, at its base configuration. Which tile that is depends on head_k_dim +# and head_v_dim (ALGORITHM.md §4.6). Known limitation: at head_k_dim = +# head_v_dim = 64 or 192, which cannot split over 16 K lanes, it is (1,1,1), +# which is register-heavy; a footprint-based fallback order is not implemented. +CONFIGURED_TILES_BY_GATE = { + gate: _auto_tiles(gate) + + tuple(t for t in _LEXICOGRAPHIC_TILES if t not in _auto_tiles(gate)) + for gate in ("gdn", "kda") +} +CONFIGURED_TILES = CONFIGURED_TILES_BY_GATE["gdn"] +KDA_CONFIGURED_TILES = CONFIGURED_TILES_BY_GATE["kda"] +assert set(CONFIGURED_TILES) == set(KDA_CONFIGURED_TILES) == set(_LEXICOGRAPHIC_TILES) # Spec ids carry the gate kind: GDN's are ``nw{}_wtk{}_bpv{}``, KDA's add a # ``kda_`` prefix, so the two can never collide in the shared registry. @@ -115,35 +88,21 @@ def spec_id_for(tile: Tuple[int, int, int], gate_kind: str) -> str: return _KDA_SPEC_ID_PREFIX + base if gate_kind == "kda" else base -def work_for(batch: int, num_v_heads: int) -> int: - """KDA's measured selection quantity.""" - return batch * num_v_heads - - -def tile_for_work(work: int, gate_kind: str = "kda") -> Tuple[int, int, int]: - if gate_kind != "kda": - raise ValueError(f"no work-keyed table for gate kind {gate_kind!r}") - for max_work, tile in _TUNED_TILES_KDA: - if max_work is None or work <= max_work: - return tile - raise AssertionError("unreachable: KDA table has an open-ended final band") - - def is_fused(req: GdnDecodeRequest) -> bool: return req.fuse_conv or req.fuse_out_norm -def auto_tile(req: GdnDecodeRequest) -> Tuple[int, int, int]: - """The tile ``auto`` prefers for ``req`` whenever it is legal. +def auto_default(req: GdnDecodeRequest) -> AutoDefault: + """The ``AUTO_DEFAULTS`` entry of ``req``'s case: its gate kind, whether it + is fused (either flag), and its state width's row (f16 reads the bf16 + row). Never batch, head count or work.""" + row = AUTO_STATE_ROW[normalize_dtype(req.state_dtype)] + return AUTO_DEFAULTS[(req.gate_kind, is_fused(req), row)] - Fused requests: ``FUSED_DEFAULT_TILE`` for both gate kinds. Unfused: GDN's - static ``DEFAULT_TILE``, KDA's work-keyed table. - """ - if is_fused(req): - return FUSED_DEFAULT_TILE - if req.gate_kind == "kda": - return tile_for_work(work_for(req.batch, req.num_v_heads), "kda") - return DEFAULT_TILE + +def auto_tile(req: GdnDecodeRequest) -> Tuple[int, int, int]: + """The tile ``auto`` prefers for ``req`` whenever it is legal.""" + return auto_default(req).tile def make_spec(req: GdnDecodeRequest, tile: Tuple[int, int, int]) -> GdnDecodeSpec: @@ -219,9 +178,9 @@ def errors(req: OperatorRequest) -> list: return [] def route_auto(req: GdnDecodeRequest) -> Tuple[bool, str]: - # ``auto`` is one preferred tile per request mode (auto_tile); when - # that tile is illegal for the request, every legal candidate stays - # admitted and priority order picks the first. + # ``auto`` is one preferred tile per case (auto_tile); when that tile + # is illegal for the request, every legal candidate stays admitted and + # priority order picks the first. wanted = auto_tile(req) if tile != wanted and _tile_is_legal(req, wanted): return False, ( @@ -230,6 +189,20 @@ def route_auto(req: GdnDecodeRequest) -> Tuple[bool, str]: ) return True, "ok" + def auto_knobs(req: GdnDecodeRequest): + # ``auto`` on its case's tile carries the case's knob stack (a knob + # the request cannot read drops; a stack the tile's space refuses + # raises); on a fallback tile it runs the tile's base configuration. + if tile != auto_tile(req): + return {} + knobs = dict(auto_default(req).knobs) + # norm_gate_once needs conv_once until ALGORITHM.md section 8, + # follow-up 8 (a NOT_YET_IMPLEMENTED rule of the validator, not + # scope), so a row that carries it drops it wherever conv_once drops. + if knobs.get("norm_gate_once") and not conv_once_applies(make_spec(req, tile)): + knobs.pop("norm_gate_once") + return knobs + def scope_check(req: GdnDecodeRequest) -> Tuple[bool, str]: # A stored tuning_id names the knob set that was canonical where it # was recorded. Where scope drops one of its knobs the id names no @@ -257,7 +230,7 @@ def scope_check(req: GdnDecodeRequest) -> Tuple[bool, str]: # tuning_knobs), the knob space walk, and the kernel validator as the # final authority. Not opt-in (``route_auto``): unlike attention, where # ``auto`` routes to untuned path candidates, GDN ``auto`` is itself one - # configuration of a registered tile (its base). + # configuration of a registered tile (``auto_knobs``: the case's stack). return make_tuned_candidate( name=name, family=FAMILY, @@ -283,6 +256,7 @@ def scope_check(req: GdnDecodeRequest) -> Tuple[bool, str]: grid=lambda spec, req: spec.launch_grid(req.batch), precheck=scope_check, route_auto=route_auto, + auto_knobs=auto_knobs, ) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx950/gdn_decode.py b/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx950/gdn_decode.py index f3e9f137bfb1..d7ec5cd26ce7 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx950/gdn_decode.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/kernels/gfx950/gdn_decode.py @@ -76,10 +76,9 @@ unfused kernel allocates no LDS (``fuse_out_norm`` adds a ``num_warps``-float LDS reduction when ``num_warps > 1``, ALGORITHM.md section 4.9; the ``conv_once`` and ``out_lds`` knobs add LDS blocks of their own, section -4.10). Dispatch selects this tile from the gfx950 GDN registry: `auto` prefers -one tile per request (``dispatch.gdn.gfx950.auto_tile``: GDN's static default, -KDA's work-keyed table, or the shared fused default for a fused request), while -an explicit `spec_id` pins a tile. +4.10). Dispatch selects this tile from the gfx950 GDN registry: `auto` runs one +static tile and knob stack per case (``dispatch.gdn.axes.AUTO_DEFAULTS``: gate +kind x fused x state width), while an explicit `spec_id` pins a tile. ``simple=True`` is the v1 reference: one workgroup per ``(sequence, value_head)``, ``head_v_dim`` threads, thread ``t`` owning state row ``t`` (the full @@ -623,7 +622,7 @@ class GdnDecodeSpec: # recurrence-only kernel at an identical work boundary. fuse_gate: bool = True wave_size: int = 64 - # GDN's dispatcher default uses these values whenever they are legal. + # The tile of GDN's plain 16-bit ``auto`` row (dispatch/gdn/axes.py). # Direct callers still get a valid general-purpose configuration. num_warps: int = 2 warp_threads_k: int = 16 diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/gdn/test_gfx950_registry.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/gdn/test_gfx950_registry.py index 8c23b04d58f0..da4974dd2448 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/gdn/test_gfx950_registry.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/gdn/test_gfx950_registry.py @@ -15,7 +15,7 @@ dispatch_gdn_decode_bases, gdn_candidates, ) -from dispatch.gdn.gfx950 import ARCH, CONFIGURED_TILES, DEFAULT_TILE, make_spec +from dispatch.gdn.gfx950 import ARCH, CONFIGURED_TILES, auto_tile, make_spec from kernels.gfx950.gdn_decode import is_valid_spec from rocke.dispatch.core import stable_json_hash @@ -119,23 +119,13 @@ def test_every_legal_pin_round_trips_and_illegal_pin_fails_loudly(): dispatch_gdn_decode(_req(algorithm="warp_tiled", spec_id="does_not_exist")) -def test_auto_is_static_default_independent_of_batch(): - outcomes = [] - for batch in (1, 16, 64, 256): - repeated = [dispatch_gdn_decode(_req(batch)) for _ in range(20)] - assert {_tile(result.spec) for result in repeated} == {DEFAULT_TILE} - assert len({result.candidate.spec_id for result in repeated}) == 1 - outcomes.append(repeated[0].candidate.spec_id) - assert len(set(outcomes)) == 1 - - def test_gdn_auto_ignores_custom_rankers_when_default_is_legal(): def reverse(request, candidates): return tuple(reversed(candidates)) result = dispatch_gdn_decode(_req(), ranker=reverse) - assert _tile(result.spec) == DEFAULT_TILE + assert _tile(result.spec) == auto_tile(_req()) def test_d64_auto_falls_back_to_a_validator_approved_candidate(): diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/gdn/test_gfx950_wiring.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/gdn/test_gfx950_wiring.py index d3e050146ebb..f0b4deba29a6 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/gdn/test_gfx950_wiring.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/gdn/test_gfx950_wiring.py @@ -13,6 +13,7 @@ import unittest from dataclasses import asdict +from itertools import product import numpy as np @@ -25,16 +26,16 @@ gdn_sweep_space, request_errors, ) +from dispatch.gdn.axes import AUTO_DEFAULTS, AUTO_STATE_ROW from dispatch.gdn.gfx950 import ( ARCH, CONFIGURED_TILES, CONFIGURED_TILES_BY_GATE, - DEFAULT_TILE, KDA_CONFIGURED_TILES, spec_id_for, - tile_for_work, ) from kernels.gfx950.gdn_decode import ( + STATE_DTYPES, gdn_decode_grid, gdn_decode_signature, is_valid_spec, @@ -77,61 +78,255 @@ def test_registry_family_is_consistent(self): self.assertEqual(cand.family, GDN_REGISTRY.family) +_NT = {"state_load_hint": "streaming", "state_store_hint": "streaming"} +_DPP = {**_NT, "dpp_reduce": True} +_FUSED_KNOBS = {"conv_once": True, "out_lds": True} +# The shipped auto table, (gate_kind, fused, state row) -> (tile, knobs). A +# default change edits this, AUTO_DEFAULTS and the ALGORITHM.md §4.6 table. +_SHIPPED = { + ("gdn", False, "bf16"): ((2, 16, 8), _DPP), + ("gdn", False, "f32"): ((2, 16, 16), {**_DPP, "interleave_cols": True}), + ("gdn", True, "bf16"): ( + (4, 16, 1), + {**_DPP, **_FUSED_KNOBS, "norm_gate_once": True, "stream_rows": True}, + ), + ("gdn", True, "f32"): ( + (8, 16, 1), + { + **_DPP, + **_FUSED_KNOBS, + "interleave_cols": True, + "stream_rows": True, + "waves_per_eu": 4, + }, + ), + ("kda", False, "bf16"): ((1, 16, 8), {**_DPP, "waves_per_eu": 4}), + ("kda", False, "f32"): ( + (1, 16, 16), + {**_DPP, "interleave_cols": True, "stream_rows": True}, + ), + ("kda", True, "bf16"): ((4, 8, 1), {**_DPP, **_FUSED_KNOBS}), + ("kda", True, "f32"): ( + (4, 8, 1), + {**_DPP, **_FUSED_KNOBS, "interleave_cols": True, "stream_rows": True}, + ), +} +_BATCHES = (1, 2, 4, 5, 8, 16, 32, 33, 64, 128, 129, 256) + + +def _case_req(batch, gate_kind, fused, state_dtype, heads=(16, 16), **kw): + return _req( + batch, + gate_kind=gate_kind, + num_k_heads=heads[0], + num_v_heads=heads[1], + state_dtype=state_dtype, + fuse_conv=fused, + fuse_out_norm=fused, + **kw, + ) + + class TestStaticSelection(unittest.TestCase): - """Dispatcher auto must use one static default rather than batch winners. + """``auto`` runs one static tile and knob stack per case (gate kind x + fused x state width), never a batch or work winner. - The shipped tile is pinned as a literal, not as ``DEFAULT_TILE``: a test - that follows the constant cannot notice the default moving. Changing a - pinned value here changes what every gfx950 GDN ``auto`` user runs, so it - needs GDN measurements (``tune.py --gate-kind gdn``) in the same change. + The shipped table is pinned as a literal, not as ``AUTO_DEFAULTS``: a test + that follows the constant cannot notice a default moving. Changing a value + here changes what every gfx950 ``auto`` user of that case runs, so it needs + a cold-memory measurement of the case in the same change. """ - _SHIPPED_TILE = (2, 16, 8) - # Known limitation (see CONFIGURED_TILES): an illegal default falls back to - # the first legal tile in product order. Pinned so a change is deliberate. + # Known limitation (see CONFIGURED_TILES): an illegal case tile falls back + # to the first legal tile in priority order, at its base configuration. + # Pinned so a change is deliberate. _FALLBACK_TILE = (1, 1, 1) - def test_default_tile_is_the_shipped_tile(self): - self.assertEqual(DEFAULT_TILE, self._SHIPPED_TILE) + def test_auto_defaults_are_the_shipped_table(self): + self.assertEqual( + {case: (d.tile, dict(d.knobs)) for case, d in AUTO_DEFAULTS.items()}, + _SHIPPED, + ) - def test_selection_is_frozen_across_head_counts_and_batches(self): + def test_f16_state_runs_the_bf16_row(self): + self.assertEqual(set(AUTO_STATE_ROW), set(STATE_DTYPES)) + self.assertEqual(AUTO_STATE_ROW["f16"], "bf16") + for (gate_kind, fused, row), _ in _SHIPPED.items(): + if row != "bf16": + continue + with self.subTest(gate=gate_kind, fused=fused): + bf16 = dispatch_gdn_decode(_case_req(8, gate_kind, fused, "bf16")) + f16 = dispatch_gdn_decode(_case_req(8, gate_kind, fused, "f16")) + self.assertEqual(f16.candidate.spec_id, bf16.candidate.spec_id) + self.assertEqual(f16.spec.knobs, bf16.spec.knobs) + self.assertEqual(f16.spec.tuning_id, bf16.spec.tuning_id) + self.assertEqual(f16.spec.kernel_spec.state_dtype, "f16") + + def test_every_case_runs_its_tile_and_knobs_at_every_batch(self): # Sharded-head deployments see head counts other than the default - # geometry, so cover them, including Hk == Hv. - for num_k_heads, num_v_heads in ( - (2, 4), - (4, 8), - (8, 16), - (16, 32), - (32, 64), - (16, 16), - ): - for batch in (1, 4, 5, 16, 32, 33, 64, 128, 129, 256): - with self.subTest(hk=num_k_heads, hv=num_v_heads, batch=batch): - result = dispatch_gdn_decode( - _req(batch, num_k_heads=num_k_heads, num_v_heads=num_v_heads) - ) - self.assertEqual(_TILE(result.spec.kernel_spec), self._SHIPPED_TILE) - self.assertEqual(result.spec.kernel_spec.gate_kind, "gdn") + # geometry, so cover them; fuse_conv needs Hk == Hv. + plain_heads = ((2, 4), (4, 8), (8, 16), (16, 32), (32, 64), (16, 16)) + fused_heads = ((4, 4), (8, 8), (16, 16), (32, 32)) + for (gate_kind, fused, row), (tile, knobs) in _SHIPPED.items(): + for heads in fused_heads if fused else plain_heads: + picks = set() + for batch in _BATCHES: + with self.subTest( + case=(gate_kind, fused, row), heads=heads, b=batch + ): + got = dispatch_gdn_decode( + _case_req(batch, gate_kind, fused, row, heads) + ) + kernel = got.spec.kernel_spec + self.assertEqual(_TILE(kernel), tile) + self.assertEqual( + got.candidate.spec_id, spec_id_for(tile, gate_kind) + ) + self.assertEqual(dict(got.spec.knobs), knobs) + for name, value in knobs.items(): + self.assertEqual(getattr(kernel, name), value, name) + self.assertEqual(kernel.gate_kind, gate_kind) + ok, why = is_valid_spec(kernel, arch=ARCH) + self.assertTrue(ok, why) + # What ran replays from the request the result carries. + self.assertEqual( + dispatch_gdn_decode(got.request).spec, got.spec + ) + picks.add(got.spec.tuning_id) + # No batch dependence: one configuration for every batch. + self.assertEqual(len(picks), 1, (gate_kind, fused, row, heads)) - def test_selected_spec_is_always_buildable(self): - for batch in (1, 4, 5, 16, 33, 64, 129, 256, 8192): - with self.subTest(batch=batch): - ok, why = is_valid_spec( - dispatch_gdn_decode(_req(batch)).spec.kernel_spec, arch=ARCH + def test_one_fused_flag_drops_the_knobs_it_cannot_read(self): + for (gate_kind, fused, row), (tile, knobs) in _SHIPPED.items(): + if not fused: + continue + for conv, norm in ((True, False), (False, True)): + with self.subTest(case=(gate_kind, row), conv=conv, norm=norm): + got = dispatch_gdn_decode( + _req( + 8, + gate_kind=gate_kind, + num_k_heads=16, + num_v_heads=16, + state_dtype=row, + fuse_conv=conv, + fuse_out_norm=norm, + ) + ) + self.assertEqual(_TILE(got.spec.kernel_spec), tile) + dropped = ( + {"norm_gate_once", "out_lds"} + if conv + else {"conv_once", "norm_gate_once"} + ) + want = {k: v for k, v in knobs.items() if k not in dropped} + self.assertEqual(dict(got.spec.knobs), want) + + def test_a_spec_id_pin_of_the_auto_tile_runs_its_base(self): + """Pins keep their meaning: a bare ``spec_id`` is the tile's base + configuration (every knob off), not the case's stack.""" + for (gate_kind, fused, row), (tile, _) in _SHIPPED.items(): + with self.subTest(case=(gate_kind, fused, row)): + req = _case_req(8, gate_kind, fused, row) + pinned = dispatch_gdn_decode( + _case_req( + 8, gate_kind, fused, row, spec_id=spec_id_for(tile, gate_kind) + ) ) - self.assertTrue(ok, why) + self.assertEqual(pinned.spec.knobs, ()) + self.assertNotEqual(pinned.spec, dispatch_gdn_decode(req).spec) + + def test_auto_is_the_case_stack_or_the_documented_fallback(self): + """Over a broad request domain: where the case's tile is legal, auto + runs it with the case's stack (the validator never refuses a stack; + the state hints are in scope everywhere, so knobs are never empty); + where it is not, auto runs the first legal tile of the gate kind's + priority order at its base configuration.""" + from dispatch.gdn.gfx950 import CONFIGURED_TILES_BY_GATE, make_spec + + fused_modes = ((False, False), (True, False), (False, True), (True, True)) + gdn_dims = tuple((d, d) for d in (64, 96, 128, 192, 256)) + ( + (128, 16), + (128, 32), + (128, 64), + (128, 256), + (256, 128), + ) + for gate_kind, dims in (("gdn", gdn_dims), ("kda", ((128, 128),))): + for (dk, dv), state_dtype, (conv, norm), heads, batch in product( + dims, + ("bf16", "f16", "f32"), + fused_modes, + ((4, 4), (16, 16), (16, 32), (48, 96)), + (1, 64), + ): + req = _req( + batch, + gate_kind=gate_kind, + num_k_heads=heads[0], + num_v_heads=heads[1], + head_k_dim=dk, + head_v_dim=dv, + state_dtype=state_dtype, + fuse_conv=conv, + fuse_out_norm=norm, + ) + if request_errors(req): + continue + case = (gate_kind, conv or norm, AUTO_STATE_ROW[state_dtype]) + tile = _SHIPPED[case][0] + with self.subTest(req=req): + got = dispatch_gdn_decode(req) + if is_valid_spec(make_spec(req, tile), arch=ARCH)[0]: + self.assertEqual(_TILE(got.spec.kernel_spec), tile) + self.assertTrue(got.spec.knobs) + else: + first = next( + t + for t in CONFIGURED_TILES_BY_GATE[gate_kind] + if is_valid_spec(make_spec(req, t), arch=ARCH)[0] + ) + self.assertEqual(_TILE(got.spec.kernel_spec), first) + self.assertEqual(got.spec.knobs, ()) - def test_supported_geometry_falls_back_when_default_is_invalid(self): + def test_supported_geometry_falls_back_when_the_case_tile_is_invalid(self): for head_k_dim in (64, 192): - for batch in (1, 256): - with self.subTest(head_k_dim=head_k_dim, batch=batch): - result = dispatch_gdn_decode(_req(batch, head_k_dim=head_k_dim)) - ok, why = is_valid_spec(result.spec.kernel_spec, arch=ARCH) - self.assertTrue(ok, why) - self.assertEqual(result.spec.kernel_spec.head_k_dim, head_k_dim) - self.assertEqual( - _TILE(result.spec.kernel_spec), self._FALLBACK_TILE - ) + for state_dtype in ("bf16", "f32"): + for fused in (False, True): + for batch in (1, 256): + with self.subTest( + head_k_dim=head_k_dim, + state=state_dtype, + fused=fused, + batch=batch, + ): + result = dispatch_gdn_decode( + _case_req( + batch, + "gdn", + fused, + state_dtype, + head_k_dim=head_k_dim, + ) + ) + kernel = result.spec.kernel_spec + ok, why = is_valid_spec(kernel, arch=ARCH) + self.assertTrue(ok, why) + self.assertEqual(kernel.head_k_dim, head_k_dim) + self.assertEqual(_TILE(kernel), self._FALLBACK_TILE) + self.assertEqual(result.spec.knobs, ()) + + def test_plain_gdn_with_a_narrow_value_dim_falls_back_to_the_fused_tile(self): + # (2,16,8) needs head_v_dim / 8 to be a multiple of 8, so DV 16 and 32 + # are illegal; the first legal tile of the GDN priority order is then + # another case's auto tile. + for head_v_dim in (16, 32): + with self.subTest(head_v_dim=head_v_dim): + result = dispatch_gdn_decode( + _case_req(8, "gdn", False, "bf16", head_v_dim=head_v_dim) + ) + self.assertEqual(_TILE(result.spec.kernel_spec), (4, 16, 1)) + self.assertEqual(result.spec.knobs, ()) class TestRequestRejection(unittest.TestCase): @@ -443,57 +638,6 @@ def test_unknown_gate_kind_is_rejected(self): self.assertTrue(any("gate_kind" in e for e in errors), errors) -class TestWorkKeyedTable(unittest.TestCase): - """The new KDA table is keyed on work = batch * num_v_heads.""" - - def test_equal_work_selects_the_same_kda_tile(self): - self.assertEqual(tile_for_work(8 * 32, "kda"), tile_for_work(32 * 8, "kda")) - self.assertEqual(tile_for_work(1 * 32, "kda"), tile_for_work(4 * 8, "kda")) - - def test_kda_dispatch_uses_work_not_batch(self): - full = dispatch_gdn_decode( - GdnDecodeRequest( - batch=4, - arch=ARCH, - gate_kind="kda", - num_k_heads=32, - num_v_heads=32, - ) - ).spec.kernel_spec - sharded = dispatch_gdn_decode( - GdnDecodeRequest( - batch=4, - arch=ARCH, - gate_kind="kda", - num_k_heads=8, - num_v_heads=8, - ) - ).spec.kernel_spec - self.assertEqual(_TILE(full), tile_for_work(4 * 32, "kda")) - self.assertEqual(_TILE(sharded), tile_for_work(4 * 8, "kda")) - - def test_tile_for_work_agrees_with_kda_dispatch(self): - for batch in (1, 8, 32, 128): - with self.subTest(batch=batch): - spec = dispatch_gdn_decode( - GdnDecodeRequest( - batch=batch, - arch=ARCH, - gate_kind="kda", - num_k_heads=32, - num_v_heads=32, - ) - ).spec.kernel_spec - self.assertEqual( - _TILE(spec), - tile_for_work(batch * spec.num_v_heads, "kda"), - ) - - def test_kda_table_is_total_over_work(self): - for work in (1, 4, 5, 128, 129, 4096, 4097, 10**6): - self.assertIsNotNone(tile_for_work(work, "kda")) - - class TestGdnAndKdaTileNamespaces(unittest.TestCase): """The shared registry keeps GDN and KDA selectable identities disjoint.""" @@ -508,8 +652,8 @@ def test_kda_spec_ids_cannot_collide_with_gdn_spec_ids(self): class TestStateDtypeDispatch(unittest.TestCase): """An f32 recurrent state is reachable from the request for both gate - kinds, and it changes only the state: the tile ``auto`` picks is the - unfused one of the same request with a 16-bit state.""" + kinds. Which tile and knobs it runs is the f32 row of its case + (``TestStaticSelection``).""" def test_every_f32_spelling_reaches_the_spec_for_both_gates(self): for gate_kind in ("gdn", "kda"): @@ -525,22 +669,6 @@ def test_every_f32_spelling_reaches_the_spec_for_both_gates(self): ("state", "ptr"), _sig_types(got.signature) ) - def test_f32_state_keeps_the_unfused_tile(self): - for gate_kind in ("gdn", "kda"): - for batch in (1, 4, 16, 64, 256): - with self.subTest(gate_kind=gate_kind, batch=batch): - narrow = dispatch_gdn_decode(_req(batch, gate_kind=gate_kind)) - wide = dispatch_gdn_decode( - _req(batch, gate_kind=gate_kind, state_dtype="f32") - ) - self.assertEqual( - _TILE(wide.spec.kernel_spec), _TILE(narrow.spec.kernel_spec) - ) - self.assertEqual(wide.candidate.spec_id, narrow.candidate.spec_id) - self.assertNotEqual( - wide.kernel_id.compile_key, narrow.kernel_id.compile_key - ) - def test_f32_is_a_state_dtype_only(self): for spelling in ("f32", "float32"): with self.subTest(spelling=spelling): @@ -571,23 +699,13 @@ def _fused_req(batch: int = 8, **kw) -> GdnDecodeRequest: class TestFusedDispatch(unittest.TestCase): """fuse_conv / fuse_out_norm requests: the flags reach the spec and the - launch ABI, auto uses the interim fused tile, only BPV=1 tiles serve, and - unfused selection does not move. - - The fused tile is pinned as a literal, not as ``FUSED_DEFAULT_TILE``, for - the reason ``TestStaticSelection`` gives. + launch ABI, auto runs the fused case's BPV=1 tile, and only BPV=1 tiles + serve. The fused tiles are pinned as literals in ``_SHIPPED``. """ - _SHIPPED_FUSED_TILE = (4, 16, 1) _MODES = ((True, False), (False, True), (True, True)) - def test_fused_default_is_the_shipped_tile(self): - from dispatch.gdn.gfx950 import FUSED_DEFAULT_TILE - - self.assertEqual(FUSED_DEFAULT_TILE, self._SHIPPED_FUSED_TILE) - def test_auto_selects_the_fused_tile_for_every_mode(self): - expected_id = {"gdn": "nw4_wtk16_bpv1", "kda": "kda_nw4_wtk16_bpv1"} for gate_kind in ("gdn", "kda"): for state_dtype in ("bf16", "f16", "f32"): for conv, norm in self._MODES: @@ -609,9 +727,11 @@ def test_auto_selects_the_fused_tile_for_every_mode(self): ) ) spec = got.spec.kernel_spec - self.assertEqual(_TILE(spec), self._SHIPPED_FUSED_TILE) + row = AUTO_STATE_ROW[state_dtype] + tile = _SHIPPED[(gate_kind, True, row)][0] + self.assertEqual(_TILE(spec), tile) self.assertEqual( - got.candidate.spec_id, expected_id[gate_kind] + got.candidate.spec_id, spec_id_for(tile, gate_kind) ) self.assertEqual( (spec.fuse_conv, spec.fuse_out_norm), (conv, norm) @@ -636,7 +756,8 @@ def test_fused_auto_ignores_head_count_and_ratio(self): ) ) self.assertEqual( - _TILE(got.spec.kernel_spec), self._SHIPPED_FUSED_TILE + _TILE(got.spec.kernel_spec), + _SHIPPED[(gate_kind, True, "bf16")][0], ) def test_fused_flags_reach_the_launch_abi(self): @@ -718,7 +839,7 @@ def test_bpv1_pins_serve_a_fused_request(self): def test_fuse_conv_with_grouped_heads_is_a_request_error(self): """Dispatch and the kernel validator report one shared rule, so the request-level rejection cannot outlive the kernel limit.""" - from dispatch.gdn.gfx950 import FUSED_DEFAULT_TILE, make_spec + from dispatch.gdn.gfx950 import make_spec from kernels.gfx950.gdn_decode import fuse_conv_heads_error rule = fuse_conv_heads_error(8, 16) @@ -729,54 +850,20 @@ def test_fuse_conv_with_grouped_heads_is_a_request_error(self): with self.subTest(gate=gate_kind): self.assertIn(rule, request_errors(req)) self.assertEqual( - is_valid_spec(make_spec(req, FUSED_DEFAULT_TILE), arch=ARCH), + is_valid_spec(make_spec(req, (4, 16, 1)), arch=ARCH), (False, rule), ) with self.assertRaises(ValueError): dispatch_gdn_decode(req) self.assertEqual(dispatch_gdn_decode_all(req), ()) - def test_unfused_selection_is_unchanged(self): - """No other KDA tile (the fused default included) answers an unfused - ``auto``: every KDA work-table entry still selects its own tile, and - GDN keeps its default.""" - for batch in (1, 4, 5, 16, 17, 64, 4096): - for num_v_heads in (8, 32): - with self.subTest(batch=batch, hv=num_v_heads): - kda = dispatch_gdn_decode( - _req( - batch, - gate_kind="kda", - num_k_heads=num_v_heads, - num_v_heads=num_v_heads, - ) - ) - band = tile_for_work(batch * num_v_heads, "kda") - self.assertEqual(kda.candidate.spec_id, spec_id_for(band, "kda")) - self.assertEqual(_TILE(kda.spec.kernel_spec), band) - gdn = dispatch_gdn_decode(_req(batch)) - kernel = gdn.spec.kernel_spec - self.assertEqual(_TILE(kernel), DEFAULT_TILE) - self.assertFalse(kernel.fuse_conv or kernel.fuse_out_norm) - - def test_a_kda_fused_tile_pin_cannot_serve_a_fused_gdn_request(self): + def test_the_kda_fused_candidate_cannot_serve_gdn(self): with self.assertRaises(PinRefused) as raised: dispatch_gdn_decode( _fused_req(gate_kind="gdn", spec_id="kda_nw4_wtk16_bpv1") ) self.assertIn("serves the 'kda' gate", str(raised.exception)) - def test_illegal_fused_default_falls_back_to_a_legal_bpv1_tile(self): - # Known limitation, as for the unfused default: head_k_dim 64 rejects - # the (4,16,1) key group, and the fallback is the first legal tile in - # product order. Pinned so a change is deliberate. - got = dispatch_gdn_decode(_fused_req(head_k_dim=64)) - ok, why = is_valid_spec(got.spec.kernel_spec, arch=ARCH) - self.assertTrue(ok, why) - self.assertEqual(_TILE(got.spec.kernel_spec), (1, 1, 1)) - kernel = got.spec.kernel_spec - self.assertTrue(kernel.fuse_conv and kernel.fuse_out_norm) - class TestRequestFieldTypes(unittest.TestCase): """The ABI-changing request flags must be real bools and the dims real diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/gdn/test_tuning_contract.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/gdn/test_tuning_contract.py index 4d19b029cba5..2d51cc72f38a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/gdn/test_tuning_contract.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/gdn/test_tuning_contract.py @@ -7,8 +7,8 @@ resolution, id uniqueness, replay and portability for representative tiles in every mode. The rest pins what is specific to this family: the knob space each mode exposes, that it holds one kernel per configuration, that ``auto`` is -each tile's base configuration, and that out-of-scope knobs are dropped while -illegal ones are refused. +one configuration of that space (its case's knob stack on the case's tile), +and that out-of-scope knobs are dropped while illegal ones are refused. """ from __future__ import annotations @@ -26,7 +26,7 @@ request_errors, ) from dispatch.gdn import gfx950, rules -from dispatch.gdn.gfx950 import ARCH +from dispatch.gdn.gfx950 import ARCH, auto_default, spec_id_for from rocke.dispatch.core import PinRefused from rocke.dispatch.tuning import sweep_level from rocke.dispatch.tuning.testing import assert_tuning_contract, representative @@ -175,29 +175,52 @@ def test_every_canonical_configuration_is_registered_once(req): @pytest.mark.parametrize("req", _MODES) -def test_auto_is_the_auto_tile_base_configuration(req): +def test_auto_is_its_case_stack_on_the_case_tile(req): + """``auto`` is a registered configuration, not a new one: the case's + knobs pinned on the case's tile give the same spec, and the knobs it + records are the stack less the knobs this request cannot read.""" auto = dispatch_gdn_decode(req) - assert auto.spec.knobs == () - assert auto.spec.tuning_id.startswith(f"{auto.candidate.spec_id}_wpe0@") + default = auto_default(req) + spec_id = spec_id_for(default.tile, req.gate_kind) + assert auto.candidate.spec_id == spec_id + wpe = default.knobs.get("waves_per_eu", 0) + assert auto.spec.tuning_id.startswith(f"{spec_id}_wpe{wpe}@") + assert dict(auto.spec.knobs).items() <= dict(default.knobs).items() kernel = auto.spec.kernel_spec - for name in ("state_load_hint", "state_store_hint"): - assert getattr(kernel, name) == "default" - for name in ( - "dpp_reduce", - "xcd_remap", - "stream_rows", - "interleave_cols", - "conv_once", - "norm_gate_once", - "out_lds", - ): - assert getattr(kernel, name) is False, name - assert kernel.waves_per_eu == 0 + for name, value in auto.spec.knobs: + assert getattr(kernel, name) == value, name + stack_pin = replace(req, spec_id=spec_id, tuning_knobs=dict(default.knobs)) + assert dispatch_gdn_decode(stack_pin).spec == auto.spec + # One of the tile's enumerated configurations. + assert auto.spec in { + r.spec for r in dispatch_gdn_decode_all(req) if r.candidate.spec_id == spec_id + } # The result's request is pinned to what ran, so it replays it. assert (auto.request.spec_id, auto.request.tuning_id) == ( auto.candidate.spec_id, auto.spec.tuning_id, ) + assert auto.request.tuning_knobs == auto.spec.knobs + assert dispatch_gdn_decode(auto.request).spec == auto.spec + + +@pytest.mark.parametrize("state_dtype", ("bf16", "f16")) +def test_norm_only_auto_drops_norm_gate_once_with_conv_once(state_dtype): + """The GDN fused bf16 row carries conv_once and norm_gate_once. On a + norm-only request conv_once is out of scope, and norm_gate_once without + it is refused by the kernel, so auto drops both and runs the rest of + the row instead of raising.""" + req = _req(gate_kind="gdn", state_dtype=state_dtype, heads=16, fuse_out_norm=True) + default = auto_default(req) + assert default.knobs.get("conv_once") and default.knobs.get("norm_gate_once") + auto = dispatch_gdn_decode(req) + assert auto.candidate.spec_id == spec_id_for(default.tile, "gdn") + expected = { + k: v + for k, v in default.knobs.items() + if k not in ("conv_once", "norm_gate_once") + } + assert dict(auto.spec.knobs) == expected assert dispatch_gdn_decode(auto.request).spec == auto.spec diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/golden/gdn_decode_gfx950_ir_sha256.json b/dnn-providers/hip-kernel-provider/rocke/library/tests/golden/gdn_decode_gfx950_ir_sha256.json index 71600b3090af..971afc1875b5 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/golden/gdn_decode_gfx950_ir_sha256.json +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/golden/gdn_decode_gfx950_ir_sha256.json @@ -2,6 +2,102 @@ "flavors": { "llvm20": { "cases": { + "auto_gdn_cv_rn_stbf16": { + "bytes": 87855, + "sha256": "0bdcdd67c91d190af2d9e5b97f99b2c4813cfff240c400f78a553a19fb316073" + }, + "auto_gdn_cv_rn_stf16": { + "bytes": 87223, + "sha256": "4a29762047316287558e0eda8517c9c4bf2ce8e50b0c5fb6739de7191731522a" + }, + "auto_gdn_cv_rn_stf32": { + "bytes": 55006, + "sha256": "19d242e9acba8748fb2d2d0f40e9f4bbbf48daa74df766c979d668ef0b05cd55" + }, + "auto_gdn_cv_stbf16": { + "bytes": 81524, + "sha256": "e72d8573454c4b16fbf744bf0c3050fc768f43ab9c63d554a2a361c7ebce793d" + }, + "auto_gdn_cv_stf32": { + "bytes": 46246, + "sha256": "30aebeccbcb1c0624333c271400d4deb905a57d3a134055ad5b1d1d8c3943fa9" + }, + "auto_gdn_plain_stbf16": { + "bytes": 27302, + "sha256": "f548cc9125aa8dea32f2b9d8296801812e5e12349772cf806c591ec1989f06f0" + }, + "auto_gdn_plain_stf16": { + "bytes": 27132, + "sha256": "78577ee842b9e075343fd70ce54949eaa9ac4ec68a7e0ba6fd7acce497fa8f4b" + }, + "auto_gdn_plain_stf32": { + "bytes": 20059, + "sha256": "23930b721dd66eb5990dbe2a5133d378d2781503a4cc7afffebfb5545a594af1" + }, + "auto_gdn_rn_gqa_stbf16": { + "bytes": 79353, + "sha256": "e820dfaa83d8f9ef1104b435db0e66dfefa7a3045ed9420f6b098debffad05e8" + }, + "auto_gdn_rn_gqa_stf32": { + "bytes": 50415, + "sha256": "65577bf9b45bc8a17383c54cecaac5cfea41131e93b62c8db2adeb4890d8e764" + }, + "auto_gdn_rn_stbf16": { + "bytes": 79351, + "sha256": "bedaa9983d4d7582982232f21ad210347a9c65065879f934370610378f5586c3" + }, + "auto_gdn_rn_stf32": { + "bytes": 50415, + "sha256": "b52c14be9202577d32e94c9c3575fdb74fc3ab1584d0e1d89ff6dea60dfd66e3" + }, + "auto_kda_cv_rn_stbf16": { + "bytes": 79667, + "sha256": "d5a0dc0343897b4f6b17da82cef49754d3126237bf92907f74e24a1cb97c24fd" + }, + "auto_kda_cv_rn_stf16": { + "bytes": 79029, + "sha256": "22c72b8236bd171f3393e0d76807480d5ce34d4f42b21006c31024dbcd7288f7" + }, + "auto_kda_cv_rn_stf32": { + "bytes": 76982, + "sha256": "7c0d4d6f50715331a90a22cc9bea8c4d37410194191479f91981b29681667621" + }, + "auto_kda_cv_stbf16": { + "bytes": 73135, + "sha256": "75f673185508635694bff75f751e03c3a81561cc045d777589ecb4457b430a5f" + }, + "auto_kda_cv_stf32": { + "bytes": 70562, + "sha256": "e82698297dbebd917f7d6eea2ea2cef2d4d923f4cbecf293b74e18386f3431a4" + }, + "auto_kda_plain_stbf16": { + "bytes": 49055, + "sha256": "7f380b034fb0dd6020e1c64a5a3d821d8201776daf50758c4548f09e8169e9fe" + }, + "auto_kda_plain_stf16": { + "bytes": 48709, + "sha256": "e5ab4dfea22e8ed82a618a6152f4d71fecc49c02324fe3b19327fd9869667d3b" + }, + "auto_kda_plain_stf32": { + "bytes": 34524, + "sha256": "92cacde2fb9ea2dc859a24d580c44902a8f8f96493143bb82a57aa0a8b227bf3" + }, + "auto_kda_rn_gqa_stbf16": { + "bytes": 86340, + "sha256": "44f0a41b760f12da5db725c88521541b9fd4b224bc1af5c41b357cb4f5e4c92d" + }, + "auto_kda_rn_gqa_stf32": { + "bytes": 85166, + "sha256": "2299629ac59c621b14ad927f737d13f6563e28a7d6b1605fe1efac313fe9b73f" + }, + "auto_kda_rn_stbf16": { + "bytes": 86338, + "sha256": "bb2372aa375c5f5e9bdf97178d4edc73a8b94c5be1d403f0206c8e51436531ca" + }, + "auto_kda_rn_stf32": { + "bytes": 85166, + "sha256": "72bf11ea92fa7c5ad97ed60debac90b40a0cfa7c0dbbc798923a241c8de3271a" + }, "default": { "bytes": 26643, "sha256": "6245c82ff931d83051bd595913c962e7dc254cb3e95ad64c3b3941dbf239b35b" @@ -1082,6 +1178,102 @@ }, "llvm22": { "cases": { + "auto_gdn_cv_rn_stbf16": { + "bytes": 87862, + "sha256": "e47cbd21909d4e40708fd51fd3fcd555e54618db0744b5ddea85dd4a055d0925" + }, + "auto_gdn_cv_rn_stf16": { + "bytes": 87230, + "sha256": "fb7a644e368a9d37cccbf9c959510c3deaea37cf95a4fd10e1b8ef24c6c50244" + }, + "auto_gdn_cv_rn_stf32": { + "bytes": 55013, + "sha256": "3860df52c817f24865d4658382292e907d8c085175e4613de47f7c2359b9e8c7" + }, + "auto_gdn_cv_stbf16": { + "bytes": 81531, + "sha256": "dae04f4e77841cb9d09ac93f2ecb49d6ba4f00c8e6e0fe4655b151bdc01d1423" + }, + "auto_gdn_cv_stf32": { + "bytes": 46253, + "sha256": "5bc2af79d45891eb2cb16877c8e3c55badbb89ad45b7e5d10dc48f77b3cbd6cf" + }, + "auto_gdn_plain_stbf16": { + "bytes": 27309, + "sha256": "404a5eb820b9d3639f059bed56cd71804fece6755149696bfe7fa691574c1b63" + }, + "auto_gdn_plain_stf16": { + "bytes": 27139, + "sha256": "24050946fe0c1612a4184bcb60fb0183c03674078ebf612f83db7f365d25a278" + }, + "auto_gdn_plain_stf32": { + "bytes": 20066, + "sha256": "59d52df00584e7d23dcd51bcab766ca206edc749e005665dae05aa897463b154" + }, + "auto_gdn_rn_gqa_stbf16": { + "bytes": 79360, + "sha256": "c5f8a7a245c726c78b2dcff350b76c4ac9863d296fca000b1ab4838a6beb425a" + }, + "auto_gdn_rn_gqa_stf32": { + "bytes": 50422, + "sha256": "f967c5e40ac4b03aa9c0f6acfc0675262eb5db42b6d0953851ad34112c0f055b" + }, + "auto_gdn_rn_stbf16": { + "bytes": 79358, + "sha256": "50632d04ff101af91f5cd095d843470d4c357640daefe813a3859a49663f7ae3" + }, + "auto_gdn_rn_stf32": { + "bytes": 50422, + "sha256": "8638692e8bc187ec8220d204f594aecc8825c068b07299ed04223b6cb0a2cab3" + }, + "auto_kda_cv_rn_stbf16": { + "bytes": 79674, + "sha256": "a9dc7ac7474e1301fe6de4c47c9b4917c12344aac7b4399b1e4c3a9dee25a509" + }, + "auto_kda_cv_rn_stf16": { + "bytes": 79036, + "sha256": "3eadd7a531b84ab1e55b63950e5acdf474a19f58f6451f407f681367adbdefa1" + }, + "auto_kda_cv_rn_stf32": { + "bytes": 76989, + "sha256": "74f4eeee883f450da85d23164b149588c04072778e2f93b8f3bc105ba843916f" + }, + "auto_kda_cv_stbf16": { + "bytes": 73142, + "sha256": "cc30c98c3af9d13a7669429251598d45c976694271928075192ae32af50b89ec" + }, + "auto_kda_cv_stf32": { + "bytes": 70569, + "sha256": "28a89af32a6b15b7b01cd7e1eb731ef312cd47005206c3cc856ba3d0be1dd656" + }, + "auto_kda_plain_stbf16": { + "bytes": 49062, + "sha256": "828fc3d857fe0f8e0b7a67e77514aa34bdf0da9785680febb33d26f4bd1a0954" + }, + "auto_kda_plain_stf16": { + "bytes": 48716, + "sha256": "ca08b6cf28886edf89aaa712ae65c7df3f6005e1581c7b84fbe9e8b2058a4249" + }, + "auto_kda_plain_stf32": { + "bytes": 34531, + "sha256": "a0623384b53593850d9b6c2863faae5af17d381393fdea764723676d086647fa" + }, + "auto_kda_rn_gqa_stbf16": { + "bytes": 86347, + "sha256": "701454c69c76a7cf0c2f581f0f0abd3da600c3f9f420a32e04433f74cdd40368" + }, + "auto_kda_rn_gqa_stf32": { + "bytes": 85173, + "sha256": "0659de53faa157919241026e1b526823602da5f03c5d97720a372df92865d274" + }, + "auto_kda_rn_stbf16": { + "bytes": 86345, + "sha256": "976d8df19990bc4158ca92795294472ca50276de352308b13d5a7697966a0b03" + }, + "auto_kda_rn_stf32": { + "bytes": 85173, + "sha256": "6e9f9de7155cd7d770f1e9d26cc06c9f87d2bc19559cc4eaad007a3308d6379a" + }, "default": { "bytes": 26650, "sha256": "6f16083ee42be522064773e7ded18ddd65c1b1e7229ca3d8b795bb12375d89d4" @@ -2162,6 +2354,102 @@ }, "llvm23": { "cases": { + "auto_gdn_cv_rn_stbf16": { + "bytes": 87926, + "sha256": "8f2154576e8b47a25d2ff376190aec1739fb9a1c450044272a25f33b3664a438" + }, + "auto_gdn_cv_rn_stf16": { + "bytes": 87294, + "sha256": "393f95d2a07bbcd2a9a84fa5fe050b5190539abfc1912861e81626782d1b265e" + }, + "auto_gdn_cv_rn_stf32": { + "bytes": 55077, + "sha256": "30f6661b04e0f3e24237772a724e88a0999254c5aaed9b53624a32007b9e08fe" + }, + "auto_gdn_cv_stbf16": { + "bytes": 81595, + "sha256": "4d342d6fcbdff5f4690bd39afc24cabcdc78b894802458d27b6aacfb2dfd9001" + }, + "auto_gdn_cv_stf32": { + "bytes": 46317, + "sha256": "cc6136b9f88afc418a32ffd5cfdfc2f9174ce9da1a84ddcb1cf304a7652fb7f2" + }, + "auto_gdn_plain_stbf16": { + "bytes": 27373, + "sha256": "763624ce90e5ee2414adf8460b24bf67556450f80a2ed32acae97cbdb0421201" + }, + "auto_gdn_plain_stf16": { + "bytes": 27203, + "sha256": "bce4a704ec89cd68f5e27c81581784044f445ddc19faca897b594b0532c9c16d" + }, + "auto_gdn_plain_stf32": { + "bytes": 20130, + "sha256": "d348fcc4b0537aff45b5e39454a52fb3612fcf7cdf7d45467644745018b08c46" + }, + "auto_gdn_rn_gqa_stbf16": { + "bytes": 79424, + "sha256": "574f276a1525dd34c1eb3c0776a4e3aa92f8d2c9a9769ba6d222b08d9e2af511" + }, + "auto_gdn_rn_gqa_stf32": { + "bytes": 50486, + "sha256": "4cefe57654d03b676eb4c60d8adadf56fa24c6fb41ba6d1ab314b7e1cad71f44" + }, + "auto_gdn_rn_stbf16": { + "bytes": 79422, + "sha256": "7e464b7cb7cca4f799dd3968c0c6406a067597bcb4a560ed0b832fa3ece1f581" + }, + "auto_gdn_rn_stf32": { + "bytes": 50486, + "sha256": "21e19efb6727c80c65aafacde18f2d20877e030d9c59705164b7ae0001c883e5" + }, + "auto_kda_cv_rn_stbf16": { + "bytes": 79738, + "sha256": "7b838f994bd94ee8aca8c5a72b98d4a34361e306fb1a12f24ed4fe4ed71e1448" + }, + "auto_kda_cv_rn_stf16": { + "bytes": 79100, + "sha256": "71bad57d11822457d5af40ded5e8a9958c9aae0e13c5e9b8d9546f838e28fe8d" + }, + "auto_kda_cv_rn_stf32": { + "bytes": 77053, + "sha256": "8397a54cc126ae301889b6d1be11a946c41b6af3def63e68568b369463d64e72" + }, + "auto_kda_cv_stbf16": { + "bytes": 73206, + "sha256": "528bebd0ee39bb62332f68749798cedd46c48ab899c5ddfbb949a20e852c8818" + }, + "auto_kda_cv_stf32": { + "bytes": 70633, + "sha256": "1733c7a00842effeef1cc95bdb86c8655dbacd80b7f0cf7fe7c5feb09db472b3" + }, + "auto_kda_plain_stbf16": { + "bytes": 49126, + "sha256": "81c1964e5ee4ae58b7cfc532ca054bf3f0f5f602571fb243945c5e65d50e7347" + }, + "auto_kda_plain_stf16": { + "bytes": 48780, + "sha256": "cd36b628958c5940381bfe9acc6bb738c1531e27145c7bd0bbcd8d323266c800" + }, + "auto_kda_plain_stf32": { + "bytes": 34595, + "sha256": "fa648732df345479d37567f5e69eeea6b9dff3c88e0d0320dc43386fe7b9d71f" + }, + "auto_kda_rn_gqa_stbf16": { + "bytes": 86411, + "sha256": "a576bb396c6e6bb049cf1e50d3011c3b823b3a6419b425fb44c8d2c198515c81" + }, + "auto_kda_rn_gqa_stf32": { + "bytes": 85237, + "sha256": "3a1d96725ca3dff06de89a0b6d64789c3bc986d460f807d96a771f6b462d22a8" + }, + "auto_kda_rn_stbf16": { + "bytes": 86409, + "sha256": "112494fc8fbf8edc4865d441969b9cef73c3c570f8bf3b86e21da84ea1c86e9c" + }, + "auto_kda_rn_stf32": { + "bytes": 85237, + "sha256": "d75e665f31aa13d621de552241901d9d57b330aa676260b722cf07e6e2d088fe" + }, "default": { "bytes": 26714, "sha256": "92bbacab90ed4cd50cb0075a449732a763bdf0d6c4582e7594e295ed3c2dec86" diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_gfx950_numeric.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_gfx950_numeric.py index 1dc40c1720ed..612897aa84e9 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_gfx950_numeric.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_gfx950_numeric.py @@ -686,13 +686,23 @@ def test_request_fields_reach_the_launch(harness, gate, state_dtype, flags, head The request fields must reach the compiled kernel AND the launch: the result's signature packs the fused tensors (``mixed_qkv`` in place of q/k/v, ``conv_state`` / ``conv_weight``, ``out_gate`` / ``norm_weight``) - from the builder's inputs. Two batches, since KDA's unfused work table may - pick a different tile for each. + from the builder's inputs. ``auto`` runs the case's tile and knob stack. """ + from dispatch.gdn import GdnDecodeRequest + from dispatch.gdn.gfx950 import auto_tile + conv, norm = flags + case = GdnDecodeRequest( + batch=1, + arch=ARCH, + gate_kind=gate, + state_dtype=state_dtype, + fuse_conv=conv, + fuse_out_norm=norm, + ) _check_dispatched_launch( harness, - expect_tile=(4, 16, 1) if conv or norm else None, + expect_tile=auto_tile(case), gate_kind=gate, num_k_heads=heads[0], num_v_heads=heads[1], @@ -715,12 +725,24 @@ def test_kda_fused_request_reaches_kimi_head_shape(harness, state_dtype, flags): """KDA's real fused-conv deployment shape has Hk == Hv == 32, not 16. The generic dispatch matrix above proves the fused ABI at a smaller MHA - shape; this pins the request-routed KDA/Kimi head count that the docs name. + shape; this pins the request-routed KDA/Kimi head count that the docs + name, on the tile ``auto`` runs for the case. """ + from dispatch.gdn import GdnDecodeRequest + from dispatch.gdn.gfx950 import auto_tile + conv, norm = flags + case = GdnDecodeRequest( + batch=1, + arch=ARCH, + gate_kind="kda", + state_dtype=state_dtype, + fuse_conv=conv, + fuse_out_norm=norm, + ) _check_dispatched_launch( harness, - expect_tile=(4, 16, 1), + expect_tile=auto_tile(case), gate_kind="kda", num_k_heads=32, num_v_heads=32, @@ -734,15 +756,27 @@ def test_kda_fused_request_reaches_kimi_head_shape(harness, state_dtype, flags): @pytest.mark.parametrize("gate", ["gdn", "kda"]) @pytest.mark.parametrize("state_dtype", ["bf16", "f32"]) def test_request_fields_reach_the_launch_without_qk_l2norm(harness, gate, state_dtype): - """A fused request with the q/k L2 norm off runs at the fused default. + """A fused request with the q/k L2 norm off runs at the fused case's + configuration. Raw (unnormalized) k lets one update grow the state to |S| ~ 18 at these inputs (about 2.5 with the norm on). There one correctly rounded bf16 store alone moves a value by up to 0.0625, more than ``TOL``, so the state error is taken relative to ``max(1, |ref|)``, like ``out_error``'s.""" + from dispatch.gdn import GdnDecodeRequest + from dispatch.gdn.gfx950 import auto_tile + + case = GdnDecodeRequest( + batch=1, + arch=ARCH, + gate_kind=gate, + state_dtype=state_dtype, + fuse_conv=True, + fuse_out_norm=True, + ) _check_dispatched_launch( harness, - expect_tile=(4, 16, 1), + expect_tile=auto_tile(case), relative_state=True, gate_kind=gate, num_k_heads=16, @@ -759,8 +793,9 @@ def test_request_fields_reach_the_launch_without_qk_l2norm(harness, gate, state_ def test_request_fields_reach_the_launch_at_the_fused_d64_fallback( harness, state_dtype ): - """GDN D64 cannot run the fused default (it needs head_k_dim % 128 == 0), - so dispatch falls back to (1, 1, 1); that tile must still be correct.""" + """GDN D64 cannot run the fused case's tile (it needs head_k_dim % 128 == + 0), so dispatch falls back to (1, 1, 1) at its base configuration; that + tile must still be correct.""" _check_dispatched_launch( harness, expect_tile=(1, 1, 1), @@ -775,6 +810,47 @@ def test_request_fields_reach_the_launch_at_the_fused_d64_fallback( ) +@requires_gfx950 +@pytest.mark.parametrize("gate", ["gdn", "kda"]) +@pytest.mark.parametrize("state_dtype", ["bf16", "f16", "f32"]) +@pytest.mark.parametrize("fused", [False, True], ids=["plain", "cv_rn"]) +def test_every_auto_configuration_matches_fp32_through_dispatch( + harness, gate, state_dtype, fused +): + """Each case's ``auto`` configuration (the tile and knob stack the golden + ``auto_*`` cases pin; f16 runs the bf16 row) is dispatched from a plain + request, compiled and launched from the dispatch result alone, and checked + against the fp32 reference at batches 1, 5, 64 and 256: output, written + state and conv state, and every untouched page.""" + from dispatch.gdn import GdnDecodeRequest, dispatch_gdn_decode + from dispatch.gdn.gfx950 import auto_default + + launchers = {} + picks = set() + for batch in (1, 5, 64, 256): + req = GdnDecodeRequest( + batch=batch, + arch=ARCH, + gate_kind=gate, + num_k_heads=16, + num_v_heads=16, + state_dtype=state_dtype, + fuse_conv=fused, + fuse_out_norm=fused, + ) + result = dispatch_gdn_decode(req) + default = auto_default(req) + kernel = result.spec.kernel_spec + assert (kernel.num_warps, kernel.warp_threads_k, kernel.blocks_per_v_dim) == ( + default.tile + ) + assert dict(result.spec.knobs) == dict(default.knobs) + assert kernel.state_dtype == state_dtype + picks.add(result.spec.tuning_id) + _launch_dispatched(harness, result, batch, launchers) + assert len(picks) == 1, picks + + @requires_gfx950 def test_mismatched_write_skip_leaves_state_untouched(harness): from kernels.gfx950.gdn_decode import GdnDecodeSpec diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_golden.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_golden.py index 2b3fe17135c5..fdfc76bddf64 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_golden.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_golden.py @@ -48,10 +48,11 @@ def _cases(): """case id -> zero-arg builder returning a KernelDef. - Covers the default spec, the reference path, and every legal registered - tile's base configuration (every knob at its default: the kernel ``auto`` - runs on that tile) for both gate kinds, so a change to any tile is - visible. Knobs are hashed by the ``knob_*`` cases: one per knob on each + Covers the default spec, the reference path, every legal registered + tile's base configuration (every knob at its default) for both gate + kinds, so a change to any tile is visible, and the configuration ``auto`` + dispatches for every case (``auto_*``), so a change to a shipped default + is too. Knobs are hashed by the ``knob_*`` cases: one per knob on each path it changes, plus the knob combinations that share LDS (``knob_conv_once_norm_gate_once``, ``knob_out_lds_norm_gate_once_kda``, ``knob_out_lds_2rounds_cv_rn_ngo``). Hashing every (tile x knob) @@ -59,7 +60,11 @@ def _cases(): covered on device, by the knob suite (every knob on every configured tile and together) and the pinned-dispatch suite, not by IR hashes. """ - from dispatch.gdn import GdnDecodeRequest, dispatch_gdn_decode_bases + from dispatch.gdn import ( + GdnDecodeRequest, + dispatch_gdn_decode, + dispatch_gdn_decode_bases, + ) from kernels.gfx950.gdn_decode import GdnDecodeSpec, build_gdn_decode def build(**overrides): @@ -203,6 +208,34 @@ def build(**overrides): cases[f"registered_{result.candidate.spec_id}"] = ( lambda spec=result.spec.kernel_spec: build_gdn_decode(spec, arch=_ARCH) ) + # What ``auto`` dispatches per case (gate kind x plain / conv + norm x + # state width; f16 runs the bf16 row on its own state type), and the row + # less the knobs a one-flag request cannot read (conv only, norm only, + # and norm only at GQA heads, the one fused mode open to Hk != Hv). + modes = {"plain": (False, False, 16), "cv_rn": (True, True, 16)} + one_flag = {"cv": (True, False, 16), "rn": (False, True, 16)} + one_flag["rn_gqa"] = (False, True, 32) + for gate_kind in ("gdn", "kda"): + for mode, (conv, norm, heads_v) in {**modes, **one_flag}.items(): + states = ("bf16", "f16", "f32") if mode in modes else ("bf16", "f32") + for state_dtype in states: + result = dispatch_gdn_decode( + GdnDecodeRequest( + batch=16, + arch=_ARCH, + gate_kind=gate_kind, + num_k_heads=16, + num_v_heads=heads_v, + state_dtype=state_dtype, + fuse_conv=conv, + fuse_out_norm=norm, + ) + ) + cases[f"auto_{gate_kind}_{mode}_st{state_dtype}"] = ( + lambda spec=result.spec.kernel_spec: build_gdn_decode( + spec, arch=_ARCH + ) + ) return cases @@ -422,7 +455,8 @@ def test_gdn_decode_ir_matches_golden(): def test_every_shipped_configuration_is_recorded(): - """A new tuned tile must arrive with a golden entry, not silently uncovered.""" + """A new registered tile or ``auto`` configuration must arrive with a golden + entry, not silently uncovered.""" import pytest from rocke.core.ir_golden import GOLDEN_FLAVORS @@ -546,7 +580,7 @@ def test_gdn_cases_carry_no_kda_marker(): gdn_ids = [cid for cid in ids if "kda" not in cid] kda_ids = [cid for cid in ids if "kda" in cid] - # The original GDN set: default, simple, no_l2norm + one per GDN tuned tile. + # The original GDN set: default, simple, no_l2norm + one per registered GDN tile. assert len(gdn_ids) >= 7, f"expected the original GDN case set, got {gdn_ids}" assert kda_ids, "the KDA gate kind is unpinned" assert not set(gdn_ids) & set(kda_ids) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_load_order.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_load_order.py index 07d010dba252..eae438bf79c0 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_load_order.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_load_order.py @@ -23,7 +23,7 @@ ``test_gdn_decode_fused.py``. Reachability: the registered GDN tiles come from dispatch, at their base -configuration (``dispatch_gdn_decode_bases``). The KDA tuned tiles at every +configuration (``dispatch_gdn_decode_bases``). The KDA auto tiles at every state dtype, the raw-gate and no-L2-norm variants, the fused modes and the knob stacks are built as specs directly here. Dispatch reaches each of those knob stacks by pin (``spec_id`` with ``tuning_knobs`` or ``tuning_id``, @@ -46,11 +46,8 @@ sys.path.insert(0, _LIB_ROOT) from dispatch.gdn import GdnDecodeRequest, dispatch_gdn_decode_bases # noqa: E402 -from dispatch.gdn.gfx950 import ( # noqa: E402 - _TUNED_TILES_KDA, - FUSED_DEFAULT_TILE, - spec_id_for, -) +from dispatch.gdn.axes import AUTO_DEFAULTS # noqa: E402 +from dispatch.gdn.gfx950 import spec_id_for # noqa: E402 from kernels.gfx950.gdn_decode import ( # noqa: E402 GdnDecodeSpec, build_gdn_decode, @@ -112,9 +109,9 @@ def _specs(): for result in dispatch_gdn_decode_bases(GdnDecodeRequest(batch=16, arch=_ARCH)): out[f"registered_{result.candidate.spec_id}"] = result.spec.kernel_spec base = GdnDecodeSpec() - kda_tiles = [tile for _, tile in _TUNED_TILES_KDA] - kda_tiles.append(FUSED_DEFAULT_TILE) - for tile in kda_tiles: + # Every KDA auto tile, on the plain path (the fused paths are below). + kda_tiles = {d.tile for (gate, _, _), d in AUTO_DEFAULTS.items() if gate == "kda"} + for tile in sorted(kda_tiles): spec_id = spec_id_for(tile, "kda") for state_dtype in ("bf16", "f16", "f32"): spec = dc.replace( @@ -125,8 +122,8 @@ def _specs(): warp_threads_k=tile[1], blocks_per_v_dim=tile[2], ) - out[f"tuned_{spec_id}_st{state_dtype}"] = spec - out[f"tuned_{spec_id}_st{state_dtype}_knobs"] = dc.replace( + out[f"auto_tile_{spec_id}_st{state_dtype}"] = spec + out[f"auto_tile_{spec_id}_st{state_dtype}_knobs"] = dc.replace( spec, **_TILE_KNOBS ) out["kda_raw_gate"] = dc.replace(base, gate_kind="kda", fuse_gate=False) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_spec.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_spec.py index fb637a7c0b74..4ecf78e9e5b6 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_spec.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_spec.py @@ -4,8 +4,9 @@ """GDN decode spec rules and emission, without a GPU. Covers the ``spec -> IR`` direction: which specs the validator admits, and that -the specs this family actually ships lower to LLVM IR -- the four tuned tiles -plus the default, the ``simple`` reference body and the ``no_l2norm`` variant. +the specs this family actually ships lower to LLVM IR -- every registered tile's +base configuration, every case's ``auto`` configuration plus the default, the +``simple`` reference body and the ``no_l2norm`` variant. That is a CURATED list, not the admitted space: the validator admits 54 legal tile combinations at the reference shape, times the dtype axis, and this file does not walk either. Lowering needs no device and no comgr, so the whole file @@ -21,6 +22,7 @@ import dataclasses as dc import unittest +from itertools import product from kernels.gfx950.gdn_decode import ( GdnDecodeSpec, @@ -383,12 +385,12 @@ def test_registered_tiles_compile(self): # kda_nw1_wtk1_bpv2 0 B 388 B # kda_nw2_wtk1_bpv1 0 B 364 B # (the KDA gate holds a decay per K channel, hence more registers). - # ``auto`` never selects them (GDN always picks DEFAULT_TILE, KDA its - # work table); they stay registered for both gate kinds because they - # can be pinned and (1,1,1) is the fallback when the auto tile is - # illegal. They may spill up to their gate kind's budget, which absorbs - # compiler drift but still catches a register-pressure blowup. Every - # other registered tile of either gate kind must not spill. + # ``auto`` never selects them on a legal case tile; they stay + # registered for both gate kinds because they can be pinned and + # (1,1,1) is the fallback when a case's tile is illegal. They may + # spill up to their gate kind's budget, which absorbs compiler drift + # but still catches a register-pressure blowup. Every other registered + # tile of either gate kind must not spill. spill_budget_bytes = {"gdn": 512, "kda": 1024} register_bound = ((1, 1, 1), (1, 1, 2), (2, 1, 1)) for gate_kind in ("gdn", "kda"): @@ -407,22 +409,37 @@ def test_registered_tiles_compile(self): else: self.assertEqual(scratch, 0) - def test_every_kda_tuned_tile_compiles(self): - from dispatch.gdn.gfx950 import _TUNED_TILES_KDA - - # f32 doubles the state registers each lane loads, so both widths run. - for state_dtype in ("bf16", "f32"): - for _, tile in _TUNED_TILES_KDA: - with self.subTest(tile=tile, state_dtype=state_dtype): - spec = dc.replace( - GdnDecodeSpec(), - gate_kind="kda", - num_warps=tile[0], - warp_threads_k=tile[1], - blocks_per_v_dim=tile[2], - state_dtype=state_dtype, - ) - self.assertEqual(_compiled_scratch_bytes(self, spec), 0) + def test_every_auto_configuration_compiles_without_spilling(self): + from dispatch.gdn import GdnDecodeRequest, dispatch_gdn_decode + + # Every case's tile and knob stack, on each state width (f32 doubles + # the state registers each lane loads), in every fused mode: the + # one-flag modes run the row less the knobs they cannot read, and + # norm only is the one fused mode open to GQA heads. + modes = ( + (False, False, 16), + (True, True, 16), + (True, False, 16), + (False, True, 16), + (False, True, 32), + ) + for gate_kind in ("gdn", "kda"): + for (conv, norm, heads_v), state_dtype in product( + modes, ("bf16", "f16", "f32") + ): + req = GdnDecodeRequest( + batch=16, + arch="gfx950", + gate_kind=gate_kind, + num_k_heads=16, + num_v_heads=heads_v, + state_dtype=state_dtype, + fuse_conv=conv, + fuse_out_norm=norm, + ) + spec = dispatch_gdn_decode(req).spec + with self.subTest(tuning_id=spec.tuning_id, state=state_dtype): + self.assertEqual(_compiled_scratch_bytes(self, spec.kernel_spec), 0) def test_default_gdn_tile_compiles_with_an_f32_state(self): spec = dc.replace(GdnDecodeSpec(), state_dtype="f32") diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_tune.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_tune.py index 11f29b684f7f..e99b8027dd0d 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_tune.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_tune.py @@ -105,10 +105,14 @@ def test_main_reports_dispatcher_default_outside_top_rows(monkeypatch, capsys): assert tune.main() == 0 output = capsys.readouterr().out - assert "dispatcher default: 6.100us default tile=(2, 16, 8) rank=2/3" in output + assert ( + "auto tile (base configuration): 6.100us default tile=(2, 16, 8) rank=2/3" + in output + ) assert "fastest legal candidate: 5.000us fast tile=(4, 16, 8)" in output - assert "default / fastest = 1.220x" in output - assert "consider DEFAULT_TILE = (4, 16, 8)" in output + assert "auto tile base / fastest = 1.220x" in output + assert "not a basis for changing AUTO_DEFAULTS" in output + assert "consider" not in output def test_main_reports_missing_gdn_default_and_continues(monkeypatch, capsys): @@ -135,28 +139,26 @@ def test_main_reports_missing_gdn_default_and_continues(monkeypatch, capsys): assert tune.main() == 1 output = capsys.readouterr().out - assert ( - "dispatcher default 'default' is NOT in the correct-and-timeable set" in output - ) + assert "auto tile 'default' is NOT in the correct-and-timeable set" in output assert "=== Hk16/Hv32 batch 2" in output - assert "manual review: retain DEFAULT_TILE" in output + assert "not a basis for changing AUTO_DEFAULTS" in output def test_report_gdn_dispatcher_default_keeps_fastest_default(capsys): tune.report_gdn_dispatcher_default([(5.0, (2, 16, 8), "default", 0.0)], "default") assert capsys.readouterr().out.splitlines() == [ - " dispatcher default: 5.000us default tile=(2, 16, 8) rank=1/1", + " auto tile (base configuration): 5.000us default tile=(2, 16, 8) rank=1/1", " fastest legal candidate: 5.000us default tile=(2, 16, 8)", - " default / fastest = 1.000x", - " manual review: retain DEFAULT_TILE", + " auto tile base / fastest = 1.000x", + " base configurations only; not a basis for changing AUTO_DEFAULTS " + "(ALGORITHM.md section 4.6)", ] -def test_main_flags_kda_cells_with_equal_work_and_different_best_tiles( - monkeypatch, capsys -): - """Equal batch * Hv must pick one tile, or the work-keyed table is invalid.""" +def test_main_flags_kda_cells_where_the_auto_tile_was_not_measured(monkeypatch, capsys): + """A KDA cell whose correct, timed rows miss the auto tile says so, and + the run continues with the other cells.""" shipped = (4, 16, 4) monkeypatch.setattr(tune, "device_is_visible", lambda: True) monkeypatch.setattr( @@ -198,16 +200,28 @@ def fake_sweep(base, batch, configs): assert tune.main() == 0 lines = capsys.readouterr().out.splitlines() - # work 128 = 4x32 (shipped wins) and 8x16 (other tile wins). - work_128 = next(line for line in lines if line.lstrip().startswith("128 ")) - assert "4x32 8x16" in work_128 - assert "TILES DISAGREE" in work_128 - # work 64 (4x16) and 256 (8x32) each have a single cell, so no flag. - assert sum("TILES DISAGREE" in line for line in lines) == 1 - assert "WARNING: work alone did not fix the best tile at 1 work value(s)." in ( - "\n".join(lines) - ) - assert ( - sum("dispatcher default (4, 16, 4) is NOT in the" in line for line in lines) - == 2 + assert sum("auto tile (4, 16, 4) is NOT in the" in line for line in lines) == 2 + + +def test_main_sweeps_kda_tiles_at_their_base_configuration(monkeypatch): + """The KDA study starts from the dispatched tile with every knob off, not + from ``auto``'s knob stack, so every tile is timed at its base.""" + monkeypatch.setattr(tune, "device_is_visible", lambda: True) + bases = [] + monkeypatch.setattr(tune, "legal_configs", lambda base: bases.append(base) or []) + monkeypatch.setattr(tune, "sweep_batch", lambda base, batch, configs: []) + monkeypatch.setattr( + "sys.argv", ["tune.py", "--gate-kind", "kda", "--batches", "16"] + ) + + tune.main() + (base,) = bases + assert base == dc.replace( + GdnDecodeSpec(), + gate_kind="kda", + num_k_heads=16, + num_v_heads=32, + num_warps=1, + warp_threads_k=16, + blocks_per_v_dim=8, ) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_kda_decode_benchmark.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_kda_decode_benchmark.py index 4dac249cfcc3..6fc4606dda4a 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_kda_decode_benchmark.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_kda_decode_benchmark.py @@ -43,12 +43,12 @@ def test_production_variants_derive_from_the_dispatched_tile(): names=("fused", "precomputed", "simple"), ) - assert result.candidate.spec_id == "kda_nw1_wtk16_bpv4" + assert result.candidate.spec_id == "kda_nw1_wtk16_bpv8" fused = variants["fused"] raw = variants["precomputed"] simple = variants["simple"] tile = (fused.num_warps, fused.warp_threads_k, fused.blocks_per_v_dim) - assert tile == (1, 16, 4) + assert tile == (1, 16, 8) assert (raw.num_warps, raw.warp_threads_k, raw.blocks_per_v_dim) == tile assert raw.fuse_gate is False assert simple.simple is True diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_kda_decode_gfx950_numeric.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_kda_decode_gfx950_numeric.py index 7e7e26937819..81d23157bc64 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_kda_decode_gfx950_numeric.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_kda_decode_gfx950_numeric.py @@ -33,7 +33,12 @@ pytestmark = pytest.mark.gpu -from dispatch.gdn.gfx950 import _TUNED_TILES_KDA # noqa: E402 +from dispatch.gdn.axes import AUTO_DEFAULTS # noqa: E402 + +# Every KDA auto tile (plain and fused cases), each built here on the plain path. +_KDA_AUTO_TILES = sorted( + {d.tile for (gate, _, _), d in AUTO_DEFAULTS.items() if gate == "kda"} +) def _device_is_gfx950() -> bool: @@ -121,11 +126,11 @@ def test_kda_warp_tiled_matches_reference(harness, batch, state_dtype): @requires_gfx950 @pytest.mark.parametrize("state_dtype", ["bf16", "f32"]) -@pytest.mark.parametrize("max_work,tile", _TUNED_TILES_KDA) -def test_every_kda_tuned_tile_is_correct(harness, max_work, tile, state_dtype): - """Every tile in the KDA table agrees with the independent reference, for - each state width. f32 doubles the state bytes each lane moves, so it - exercises a different load/store split.""" +@pytest.mark.parametrize("tile", _KDA_AUTO_TILES) +def test_every_kda_auto_tile_is_correct(harness, tile, state_dtype): + """Every KDA auto tile's base configuration agrees with the independent + reference, for each state width. f32 doubles the state bytes each lane + moves, so it exercises a different load/store split.""" num_warps, warp_threads_k, blocks_per_v_dim = tile spec = _kda( num_warps=num_warps, @@ -135,29 +140,28 @@ def test_every_kda_tuned_tile_is_correct(harness, max_work, tile, state_dtype): ) out_err, state_err = harness["check"](spec, batch=16) - assert out_err <= harness["TOL"], ( - f"KDA tile {tile} (max_work={max_work}, state {state_dtype}) " - f"output error {out_err:.3e}" - ) - assert state_err <= harness["TOL"], ( - f"KDA tile {tile} (max_work={max_work}, state {state_dtype}) " - f"state error {state_err:.3e}" - ) + assert ( + out_err <= harness["TOL"] + ), f"KDA tile {tile} (state {state_dtype}) output error {out_err:.3e}" + assert ( + state_err <= harness["TOL"] + ), f"KDA tile {tile} (state {state_dtype}) state error {state_err:.3e}" @requires_gfx950 @pytest.mark.parametrize( - "batch,expected_spec_id,expected_tile", + "state_dtype,expected_spec_id,expected_tile", [ - (1, "kda_nw4_wtk16_bpv4", (4, 16, 4)), - (8, "kda_nw1_wtk16_bpv4", (1, 16, 4)), - (32, "kda_nw2_wtk16_bpv1", (2, 16, 1)), + ("bf16", "kda_nw1_wtk16_bpv8", (1, 16, 8)), + ("f32", "kda_nw1_wtk16_bpv16", (1, 16, 16)), ], ) -def test_kda_dispatch_band_launches_selected_kernel( - harness, batch, expected_spec_id, expected_tile +@pytest.mark.parametrize("batch", [1, 8, 32]) +def test_kda_dispatch_launches_the_auto_kernel( + harness, batch, state_dtype, expected_spec_id, expected_tile ): - """Exercise request → dispatch → selected tile → compile → launch → oracle.""" + """Exercise request → dispatch → selected tile → compile → launch → oracle. + The tile is the case's at every batch.""" from dispatch.gdn import GdnDecodeRequest, dispatch_gdn_decode from rocke.helpers.compile import compile_kernel from rocke.runtime.launcher import KernelLauncher, LaunchConfig, no_fence @@ -169,6 +173,7 @@ def test_kda_dispatch_band_launches_selected_kernel( gate_kind="kda", num_k_heads=32, num_v_heads=32, + state_dtype=state_dtype, ) ) spec = result.spec.kernel_spec diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_kda_decode_resources.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_kda_decode_resources.py index db247fb756b9..86c0bf1d6c35 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_kda_decode_resources.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_kda_decode_resources.py @@ -13,10 +13,13 @@ from dispatch.gdn import GdnDecodeRequest, dispatch_gdn_decode ARCH = "gfx950" +# Every KDA auto case: (state_dtype, fused, spec_id, tile). Each case runs one +# configuration at every batch, so one batch per case is enough. CASES = ( - (1, "kda_nw4_wtk16_bpv4", (4, 16, 4)), - (8, "kda_nw1_wtk16_bpv4", (1, 16, 4)), - (32, "kda_nw2_wtk16_bpv1", (2, 16, 1)), + ("bf16", False, "kda_nw1_wtk16_bpv8", (1, 16, 8)), + ("f32", False, "kda_nw1_wtk16_bpv16", (1, 16, 16)), + ("bf16", True, "kda_nw4_wtk8_bpv1", (4, 8, 1)), + ("f32", True, "kda_nw4_wtk8_bpv1", (4, 8, 1)), ) @@ -60,18 +63,23 @@ def test_scratch_gate_rejects_nonzero_metadata(): ) -@pytest.mark.parametrize("batch,expected_spec_id,expected_tile", CASES) -def test_dispatched_kda_tile_is_scratch_free(batch, expected_spec_id, expected_tile): - """Compile each production candidate and reject register spills.""" +@pytest.mark.parametrize("state_dtype,fused,expected_spec_id,expected_tile", CASES) +def test_dispatched_kda_tile_is_scratch_free( + state_dtype, fused, expected_spec_id, expected_tile +): + """Compile each production configuration and reject register spills.""" result = dispatch_gdn_decode( GdnDecodeRequest( - batch=batch, + batch=8, arch=ARCH, gate_kind="kda", num_k_heads=32, num_v_heads=32, head_k_dim=128, head_v_dim=128, + state_dtype=state_dtype, + fuse_conv=fused, + fuse_out_norm=fused, ) ) spec = result.spec.kernel_spec diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/instances/gdn.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/instances/gdn.md index 5c7efe130f66..ed904ce525cd 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/instances/gdn.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/instances/gdn.md @@ -88,8 +88,8 @@ performance knobs (`state_load_hint`, `state_store_hint`, `dpp_reduce`, `xcd_remap`, `stream_rows`, `interleave_cols`, `conv_once`, `norm_gate_once`, `out_lds`, `waves_per_eu`) change how the warp-tiled body computes (the two state hints and `waves_per_eu` also apply to the reference body); each -defaults to the code emitted before it existed, and `auto` keeps every one -off. Their rules live in +defaults to the code emitted before it existed, and `auto` turns on a measured +stack of them per case. Their rules live in [`ALGORITHM.md` §4.10](../../../library/builders/gfx950/gdn/ALGORITHM.md). `is_valid_spec(spec, arch)` rejects unbuildable configurations before IR @@ -151,27 +151,22 @@ GDN and KDA each register the same 180 stable tile identities for KDA). `is_valid_spec()` filters that configured space per request; the default D128 request admits 54. KDA accepts only D128 today, so its other 126 tiles stay registered but unselectable until that limit is lifted. -Production `auto` deterministically prefers GDN's `(num_warps=2, -warp_threads_k=16, blocks_per_v_dim=8)` whenever it is legal. It never measures -at runtime and does not select by batch. An explicit `spec_id`, such as -`nw4_wtk16_bpv8`, selects an exact legal tile for benchmarking or replay. +Production `auto` runs one static configuration per case: gate kind x fused +(either flag) x state width, a tile plus a knob stack (an `f16` state uses the +`bf16` row). It never measures at runtime and does not select by batch, head +count or `batch * num_v_heads`. Fused cases use `blocks_per_v_dim == 1` tiles, +which fused requests need. The table, its provenance and the fallback when a +case's tile is illegal live once in +[`ALGORITHM.md` §4.6](../../../library/builders/gfx950/gdn/ALGORITHM.md#46-registry-and-tile-selection). +An explicit `spec_id`, such as `nw4_wtk16_bpv8`, selects an exact legal tile for +benchmarking or replay. Each registered tile is a tuned candidate (the shared `rocke.dispatch.tuning` template) whose knob space holds the performance knobs: every (tile x knob) configuration is listed by `dispatch_gdn_decode_all` and pinnable by -`spec_id` plus `tuning_id` / `tuning_knobs`, and `auto` runs the tile's base -configuration. The sweep levels and the pin and scope rules are in -[`ALGORITHM.md` §4.6](../../../library/builders/gfx950/gdn/ALGORITHM.md#46-registry-and-tile-selection). - -KDA `auto` keeps its separately measured table keyed by -`work = batch * num_v_heads`, so tensor-parallel head sharding maps to the same -key as an equivalent amount of batch work. Re-measure KDA with -`library/builders/gfx950/gdn/tune.py`; exact measurements live outside the -public source tree. - -Fused requests (`fuse_conv` and/or `fuse_out_norm`) need `blocks_per_v_dim == -1`, so `auto` prefers an interim static fused tile for both gate kinds; its -choice, provenance and fallback live once in +`spec_id` plus `tuning_id` / `tuning_knobs`. A bare `spec_id` pin runs the +tile's base configuration, and `auto`'s configuration is one of these. The +sweep levels and the pin and scope rules are in [`ALGORITHM.md` §4.6](../../../library/builders/gfx950/gdn/ALGORITHM.md#46-registry-and-tile-selection). ## Dispatch @@ -192,7 +187,7 @@ rejects. gfx950 only. Through dispatch, `bf16` and `f16` activation dtypes and `bf16`, `f16` and `f32` state dtypes are supported and need not match, for both gate -kinds; an `f32` state does not change the tile `auto` picks. `fuse_conv` and +kinds; the state width selects its case's `auto` row. `fuse_conv` and `fuse_out_norm` are request fields (real bools; `fuse_conv` also needs `num_k_heads == num_v_heads`), and the result's signature is the fused ABI, so the launch packs the fused tensors from `prepare()`. Head geometry is diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/instances/index.md b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/instances/index.md index d0cc94367ff9..f3e68475aed3 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/instances/index.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/dsl_docs/instances/index.md @@ -146,11 +146,11 @@ Runtime entry points: `dispatch_gdn_decode(GdnDecodeRequest(...))` (single-token Linear attention carries a fixed-size recurrent state per value head instead of re-reading past tokens, so cost per token does not grow with sequence length. GDN ships a single-token decode kernel and a split chunkwise prefill mode; gfx950. -GDN decode uses the static `(2, 16, 8)` dispatcher default whenever it is legal; -batch changes its grid, not its auto tile. GDN prefill picks `value_splits` per -`batch_heads`. Both tune the same tension -- splitting a head's value dimension -across workgroups buys parallelism when the natural grid starves, and costs -redundant tile reads once it does not. +GDN decode runs one static `auto` configuration per case (gate kind x fused x +state width; `instances/gdn.md`); batch changes its grid, not its configuration. +GDN prefill picks `value_splits` per `batch_heads`. Both tune the same tension +-- splitting a head's value dimension across workgroups buys parallelism when +the natural grid starves, and costs redundant tile reads once it does not. ## Small Ops diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/dispatch/ARCHITECTURE.md b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/dispatch/ARCHITECTURE.md index 0b094c0ddc93..d8ad691f201b 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/dispatch/ARCHITECTURE.md +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/dispatch/ARCHITECTURE.md @@ -2227,20 +2227,32 @@ resolution, sweeping and sampling. Normalize device or process state into the request first; do not probe it from `base`. Attention's explicit tuning candidates therefore use `_tuning_problem`, while production path routing uses the live-CU-aware `_problem`. The candidate is opt-in by default. A family -whose `auto` is itself one configuration of a registered variant (GDN decode: -each tile's base configuration) passes `route_auto(req)` instead: the -candidate is then not opt-in, resolves an `auto` request as if it pinned the -candidate's `algorithm` and `spec_id`, and `route_auto` says whether it takes -a request whose `spec_id` is `auto`. +whose `auto` is itself one configuration of a registered variant passes +`route_auto(req)` instead: the candidate is then not opt-in, resolves an +`auto` request as if it pinned the candidate's `algorithm` and `spec_id`, and +`route_auto` says whether it takes a request whose `spec_id` is `auto`. Its +`auto` runs the space's default unless the family also passes +`auto_knobs(req)`: the knobs `auto` runs on that candidate, resolved as a +`tuning_knobs` pin is (a knob the request cannot read drops). An empty mapping +runs the default; knobs that do not resolve on a request whose default does +raise `RuntimeError` with the reason, never a quiet fallback; `auto_knobs` +without `route_auto` is a construction error. An `auto` request that carries +its own tuning pin (a `tuning_id` other than `auto`, or `tuning_knobs`) +resolves that pin; the hook never replaces it. GDN decode uses both: `auto` is +the case's knob stack on the case's tile (`library/dispatch/gdn/axes.py`, +`AUTO_DEFAULTS`), and a fallback tile's default where the case's tile is +illegal. **The test.** Call `assert_tuning_contract(candidate, requests, other_requests=..., default_knobs=..., refused_knobs=...)` for a representative set of candidates (`representative(candidates, prefixes)` -picks them by name). It checks that `auto` is the first production spec, ids +picks them by name). It checks that a pinned `auto` (the candidate's +`spec_id`, no knobs: the space's default) is the first production spec, ids are unique, `(id, knobs)`, knobs alone, a bare id, and the stored dispatch request all reselect the spec, mismatched knobs are refused, default-valued knobs canonicalize to the default, and `config_key` survives a change of -problem. +problem. It does not exercise a production `auto` with `auto_knobs`; the +family's own dispatch tests cover that. Worked examples: the toy family in `platform/tests/dispatch/dispatch_tests/tuning/test_tuning_template.py` (every diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/dispatch/tuning/candidate.py b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/dispatch/tuning/candidate.py index 48f17d5f5adb..a7fc4244ee10 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/dispatch/tuning/candidate.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/python/rocke/dispatch/tuning/candidate.py @@ -4,17 +4,20 @@ By default it is opt-in: admitted only when the request names both its ``algorithm`` and ``spec_id``. A family whose ``auto`` is itself one -configuration of a registered variant passes ``route_auto`` and gets a -candidate that also answers ``auto``. Its configuration comes from the -request's ``tuning_knobs`` (rebuilt directly), else its ``tuning_id`` -(``auto``: the space's default; otherwise a search of the space). ``support`` -and ``select_spec`` share one resolution per request. +configuration of a registered variant passes ``route_auto`` (and optionally +``auto_knobs``) and gets a candidate that also answers ``auto``. Its +configuration comes from the request's ``tuning_knobs`` (rebuilt directly), +else its ``tuning_id`` (``auto``: on an ``auto`` ``spec_id`` with no tuning +pin, the ``auto_knobs`` stack if given, else the space's default; otherwise a +search of the space). ``support`` and +``select_spec`` share one resolution per request. """ from __future__ import annotations +from dataclasses import replace from functools import lru_cache -from typing import Any, Callable, Optional, Sequence, Tuple +from typing import Any, Callable, Mapping, Optional, Sequence, Tuple from ..core import ( Capability, @@ -37,6 +40,12 @@ def explicitly_pinned(req, algorithm: str, spec_id: str) -> bool: ) and normalize_selector(req.algorithm) == normalize_selector(algorithm) +def carries_tuning_pin(req) -> bool: + """Whether ``req`` pins a configuration itself: a ``tuning_id`` other + than ``auto`` (blank counts as ``auto``) or any ``tuning_knobs``.""" + return (req.tuning_id.strip() or "auto") != "auto" or bool(dict(req.tuning_knobs)) + + def resolve_pinned( req, *, @@ -54,10 +63,10 @@ def resolve_pinned( A pin that does not resolve is refused with the reason; the candidate never substitutes another configuration. """ + if not carries_tuning_pin(req): + return default() wanted = req.tuning_id.strip() or "auto" knobs = dict(req.tuning_knobs) - if wanted == "auto" and not knobs: - return default() spec, why = canonicalize(knobs) if wanted == "auto": return spec, why @@ -110,6 +119,7 @@ def make_tuned_candidate( block: Callable[[Any], Tuple[int, int, int]] = lambda spec: spec.launch_block(), precheck: Optional[Callable[[Any], Tuple[bool, str]]] = None, route_auto: Optional[Callable[[Any], Tuple[bool, str]]] = None, + auto_knobs: Optional[Callable[[Any], Mapping[str, Any]]] = None, ) -> KernelCandidate: """A candidate over ``space``, opt-in unless ``route_auto`` is given. @@ -127,8 +137,18 @@ def make_tuned_candidate( or name this candidate is resolved as if it pinned both, and for one whose ``spec_id`` is ``auto``, ``route_auto`` says whether this candidate takes it (``(False, why)`` leaves it to another candidate). Without it the candidate - is opt-in. + is opt-in. ``auto_knobs(req)`` (with ``route_auto`` only) gives the knobs an + ``auto`` request with no tuning pin (``tuning_id`` ``auto``, no + ``tuning_knobs``) runs on this candidate, resolved as ``tuning_knobs`` are + (a knob the request cannot read drops); an empty mapping runs the space's + default. A request that carries its own tuning pin resolves it as given. + Knobs that do not resolve on a request whose space default does are a + family data error and raise ``RuntimeError`` with the reason; ``auto`` + never quietly runs another configuration. Without ``auto_knobs``, ``auto`` + runs the space's default. """ + if auto_knobs is not None and route_auto is None: + raise ValueError(f"{name}: auto_knobs needs route_auto") def routed(req) -> Tuple[Optional[Any], str]: """``req`` pinned to this candidate, or ``(None, why)`` if it does not @@ -146,6 +166,22 @@ def routed(req) -> Tuple[Optional[Any], str]: ok, why = route_auto(req) if not ok: return None, why + probe = opt_in_probe(req, candidate) + pinned = carries_tuning_pin(req) + knobs = auto_knobs(req) if auto_knobs is not None and not pinned else None + if knobs: + stacked = replace(probe, tuning_knobs=dict(knobs)) + found, why = resolve(stacked) + if found is not None: + return stacked, "ok" + # A request the space cannot build at all is refused below with + # the base's reason; one it can build must take the stack. + if resolve(probe)[0] is not None: + raise RuntimeError( + f"{name}: auto knobs {dict(knobs)!r} do not resolve on " + f"a request whose default configuration does: {why}" + ) + return probe, "ok" return opt_in_probe(req, candidate), "ok" @lru_cache(maxsize=32) diff --git a/dnn-providers/hip-kernel-provider/rocke/platform/tests/dispatch/dispatch_tests/tuning/test_tuning_template.py b/dnn-providers/hip-kernel-provider/rocke/platform/tests/dispatch/dispatch_tests/tuning/test_tuning_template.py index f7602f5c9358..a56f961f27d3 100644 --- a/dnn-providers/hip-kernel-provider/rocke/platform/tests/dispatch/dispatch_tests/tuning/test_tuning_template.py +++ b/dnn-providers/hip-kernel-provider/rocke/platform/tests/dispatch/dispatch_tests/tuning/test_tuning_template.py @@ -530,6 +530,47 @@ def test_without_route_auto_the_candidate_stays_opt_in(self): self.assertFalse(ok) self.assertIn("opt-in", why) + def _auto_knobs_candidate(self, tile, knobs): + return _candidate( + tile, + route_auto=lambda req: (True, "ok"), + auto_knobs=lambda req: knobs, + ) + + def test_auto_knobs_run_on_auto_only(self): + for knobs, want in (({"unroll": 2}, {"unroll": 2}), ({}, {})): + with self.subTest(knobs=knobs): + candidate = self._auto_knobs_candidate(64, knobs) + auto = candidate.select_spec(ToyRequest(m=1024)) + self.assertEqual(dict(auto.knobs), want) + pinned = ToyRequest(m=1024, spec_id=candidate.spec_id) + self.assertEqual(candidate.select_spec(pinned).knobs, ()) + + def test_auto_knobs_that_do_not_resolve_raise_with_the_reason(self): + # tile128 x unroll 4 exceeds the toy register budget, while tile128's + # default builds: auto must not quietly run the default instead. + candidate = self._auto_knobs_candidate(128, {"unroll": 4}) + with self.assertRaisesRegex(RuntimeError, "register budget"): + candidate.admits(ToyRequest(m=1024)) + # A request the space cannot build at all is refused, not raised. + ok, why = candidate.admits(ToyRequest(m=0)) + self.assertFalse(ok) + self.assertIn("m must be positive", why) + + def test_auto_knobs_need_route_auto(self): + with self.assertRaisesRegex(ValueError, "auto_knobs needs route_auto"): + _candidate(64, auto_knobs=lambda req: {"unroll": 2}) + + def test_auto_knobs_never_replace_the_callers_tuning_pin(self): + # A family whose request check lets an auto request carry its own + # knobs: the hook's stack must not silently replace them. + candidate = self._auto_knobs_candidate(64, {"unroll": 2}) + mine = candidate.select_spec(ToyRequest(m=1024, tuning_knobs={"unroll": 4})) + self.assertEqual(dict(mine.knobs), {"unroll": 4}) + # A stored tuning_id alone is a pin too. + by_id = candidate.select_spec(ToyRequest(m=1024, tuning_id=mine.tuning_id)) + self.assertEqual(by_id, mine) + if __name__ == "__main__": unittest.main() From 0b2d37a2892488e153a3d2bdf4e1bee77ef3084e Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Wed, 28 Oct 2026 02:26:31 -0400 Subject: [PATCH 2/3] fix(rocke): run auto's waves_per_eu only at the measured head dims The gfx950 GDN fused f32 auto row sets waves_per_eu=4, measured at head_k_dim = head_v_dim = 128 only. At a wider key dim the state no longer fits the capped registers and the kernel spills. auto_knobs now drops that knob for other head dims and keeps the rest of the row. Adds a dispatch test for the dropped knob and a compiled-scratch test off the measured dims. --- .../library/builders/gfx950/gdn/ALGORITHM.md | 14 +++++++----- .../rocke/library/dispatch/gdn/axes.py | 6 +++++ .../rocke/library/dispatch/gdn/gfx950.py | 9 +++++++- .../tests/dispatch/gdn/test_gfx950_wiring.py | 15 +++++++++++++ .../library/tests/test_gdn_decode_spec.py | 22 +++++++++++++++++++ 5 files changed, 59 insertions(+), 7 deletions(-) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/gdn/ALGORITHM.md b/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/gdn/ALGORITHM.md index 85b18c35fe47..479f0b1be62f 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/gdn/ALGORITHM.md +++ b/dnn-providers/hip-kernel-provider/rocke/library/builders/gfx950/gdn/ALGORITHM.md @@ -537,9 +537,9 @@ reference before it is timed, and removes the knobs and values that do not help. **`auto` defaults.** Production `auto` runs one static configuration per *case*: gate kind × fused (either `fuse_conv` or `fuse_out_norm`) × state width. Each is a tile plus a knob stack on top of that tile's base, kept as data in `dispatch/gdn/axes.py` (`AUTO_DEFAULTS`). Batch, head -count and `batch × num_v_heads` never select it: they change the grid only. Every case turns on -streaming (nontemporal) state loads and stores; the table lists the rest. A knob not listed stays -off (`waves_per_eu` `0`). +count and `batch × num_v_heads` never select it: they change the grid only. Head dims change it +only through `waves_per_eu` (below). Every case turns on streaming (nontemporal) state loads and +stores; the table lists the rest. A knob not listed stays off (`waves_per_eu` `0`). | Gate | Mode | State | Tile `(NW, WTK, BPV)` | Knobs besides the streaming state hints | | --- | --- | --- | --- | --- | @@ -576,9 +576,11 @@ f32 the rule picks `(16, 16, 1)` with `interleave_cols`, `conv_once`, `out_lds`, Not measured: a fused request with one flag (it runs its row with the other flag's knobs dropped, below; e.g. `fuse_out_norm` alone at `Hk ≠ Hv`, where `fuse_conv` is not allowed), other -head dims where the case tile is legal (e.g. GDN at `DK = 256`, which runs the row; with -`waves_per_eu=4` the GDN fused f32 row spills there), an `f16` state (it moves the same bytes as -bf16, so it runs the bf16 row), f16 I/O and `use_qk_l2norm` off (each runs its row), and gfx942. +head dims where the case tile is legal (e.g. GDN at `DK = 256`, which runs its row without +`waves_per_eu=4`: that knob was measured only at `DK = DV = 128`, and with it the GDN fused f32 +row spills at a wider key dim; the row's other knobs are not timed there), an `f16` state (it +moves the same bytes as bf16, so it runs the bf16 row), f16 I/O and `use_qk_l2norm` off (each +runs its row), and gfx942. To revisit a case, sweep its tiles × knobs cold (`tune.py` times tiles at their base configuration only; an in-tree tiles × knobs sweep is a follow-up, §8), then change its row in `AUTO_DEFAULTS`, the literal table in `test_gfx950_wiring.py` and this table together. diff --git a/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/axes.py b/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/axes.py index 129ee6c14932..6f7108763cbe 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/axes.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/axes.py @@ -107,6 +107,12 @@ def _auto(tile: Tuple[int, int, int], **knobs) -> AutoDefault: {"bf16": "bf16", "f16": "bf16", "f32": "f32"} ) +# ``waves_per_eu`` caps the registers per wave and was measured only at these +# head dims (head_k_dim, head_v_dim). A request with other head dims runs its +# row without it (``auto_knobs``): at a wider key dim the state no longer fits +# and the fused f32 GDN row spills. Every other knob of the row is kept. +AUTO_WAVES_PER_EU_HEAD_DIMS: Tuple[int, int] = (128, 128) + # The static ``auto`` default per case: (gate_kind, fused, state row), where # fused means fuse_conv or fuse_out_norm. One entry per case, never keyed on # batch, head count or work: the batch only changes the grid. Chosen on gfx950 diff --git a/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/gfx950.py b/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/gfx950.py index 196347c226bc..b921b14f2f40 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/gfx950.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/dispatch/gdn/gfx950.py @@ -35,7 +35,12 @@ from rocke.dispatch.core import Capability, KernelCandidate, OperatorRequest from rocke.dispatch.tuning import make_tuned_candidate -from .axes import AUTO_DEFAULTS, AUTO_STATE_ROW, AutoDefault +from .axes import ( + AUTO_DEFAULTS, + AUTO_STATE_ROW, + AUTO_WAVES_PER_EU_HEAD_DIMS, + AutoDefault, +) from .common import ( FAMILY, GDN_ABI_VERSION, @@ -201,6 +206,8 @@ def auto_knobs(req: GdnDecodeRequest): # scope), so a row that carries it drops it wherever conv_once drops. if knobs.get("norm_gate_once") and not conv_once_applies(make_spec(req, tile)): knobs.pop("norm_gate_once") + if (req.head_k_dim, req.head_v_dim) != AUTO_WAVES_PER_EU_HEAD_DIMS: + knobs.pop("waves_per_eu", None) return knobs def scope_check(req: GdnDecodeRequest) -> Tuple[bool, str]: diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/gdn/test_gfx950_wiring.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/gdn/test_gfx950_wiring.py index f0b4deba29a6..028a4de16781 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/gdn/test_gfx950_wiring.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/dispatch/gdn/test_gfx950_wiring.py @@ -196,6 +196,21 @@ def test_every_case_runs_its_tile_and_knobs_at_every_batch(self): # No batch dependence: one configuration for every batch. self.assertEqual(len(picks), 1, (gate_kind, fused, row, heads)) + def test_waves_per_eu_is_dropped_off_the_measured_head_dims(self): + # The row's waves_per_eu was measured at DK = DV = 128 only; elsewhere + # it caps the registers below what the state needs and the kernel + # spills, so auto runs the rest of the row without it. + knobs = _SHIPPED[("gdn", True, "f32")][1] + self.assertIn("waves_per_eu", knobs) + want = {k: v for k, v in knobs.items() if k != "waves_per_eu"} + for dk, dv in ((256, 128), (256, 256), (128, 256)): + with self.subTest(dk=dk, dv=dv): + got = dispatch_gdn_decode( + _case_req(8, "gdn", True, "f32", head_k_dim=dk, head_v_dim=dv) + ) + self.assertEqual(dict(got.spec.knobs), want) + self.assertEqual(got.spec.kernel_spec.waves_per_eu, 0) + def test_one_fused_flag_drops_the_knobs_it_cannot_read(self): for (gate_kind, fused, row), (tile, knobs) in _SHIPPED.items(): if not fused: diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_spec.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_spec.py index 4ecf78e9e5b6..32dd995521a4 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_spec.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_spec.py @@ -441,6 +441,28 @@ def test_every_auto_configuration_compiles_without_spilling(self): with self.subTest(tuning_id=spec.tuning_id, state=state_dtype): self.assertEqual(_compiled_scratch_bytes(self, spec.kernel_spec), 0) + def test_auto_does_not_spill_off_the_measured_head_dims(self): + # The sweep ran at DK = DV = 128. The fused f32 GDN row's + # waves_per_eu caps the registers; with a wider key dim the state no + # longer fits and the kernel spills, so auto must not run it there. + from dispatch.gdn import GdnDecodeRequest, dispatch_gdn_decode + + for dk, dv in ((256, 128), (256, 256), (128, 256)): + req = GdnDecodeRequest( + batch=16, + arch="gfx950", + num_k_heads=16, + num_v_heads=16, + head_k_dim=dk, + head_v_dim=dv, + state_dtype="f32", + fuse_conv=True, + fuse_out_norm=True, + ) + spec = dispatch_gdn_decode(req).spec + with self.subTest(dk=dk, dv=dv, tuning_id=spec.tuning_id): + self.assertEqual(_compiled_scratch_bytes(self, spec.kernel_spec), 0) + def test_default_gdn_tile_compiles_with_an_f32_state(self): spec = dc.replace(GdnDecodeSpec(), state_dtype="f32") self.assertEqual(_compiled_scratch_bytes(self, spec), 0) From 436c6d0affe817299082e6eecd01bfd9fa60df92 Mon Sep 17 00:00:00 2001 From: Aviral Goel Date: Wed, 28 Oct 2026 02:37:27 -0400 Subject: [PATCH 3/3] test(rocke): label the tune test's stub timings as synthetic The dispatcher-default report test stubs the sweep with invented timings so each printed label can be checked against its own row. Say so next to the stub, so the values are not read as measurements. --- .../rocke/library/tests/test_gdn_decode_tune.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_tune.py b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_tune.py index e99b8027dd0d..9c132b9790a8 100644 --- a/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_tune.py +++ b/dnn-providers/hip-kernel-provider/rocke/library/tests/test_gdn_decode_tune.py @@ -84,6 +84,8 @@ def test_main_reports_dispatcher_default_outside_top_rows(monkeypatch, capsys): "dispatch_gdn_decode_bases", lambda request: (object(), object(), object()), ) + # Synthetic fixture timings, not measurements: no kernel runs here. They + # only need to differ, so each printed label must come from its own row. monkeypatch.setattr( tune, "sweep_registry_batch",