Device verification: close the last open items, and catch a shipped Krylov regression

benchmarks/baselines/nvidia_sm120_solver_krylov_performance.json

@@ -10,8 +10,8 @@
       "mode": "cooperative_grid:end_to_end",
       "selected_route": "cooperative_grid",
       "timing_domain": "end_to_end",
-      "median_ms": 1.480445,
-      "max_latency_ms": 3.701112,
+      "median_ms": 1.57365,
+      "max_latency_ms": 3.934125,
       "reduction_ctas": 3,
       "iterations": 8,
       "correctness_gate": "known_solution_plus_fp32_true_residual",
@@ -24,8 +24,8 @@
       "mode": "cooperative_grid:device_event",
       "selected_route": "cooperative_grid",
       "timing_domain": "device_event",
-      "median_ms": 0.289926,
-      "max_latency_ms": 0.724816,
+      "median_ms": 0.292467,
+      "max_latency_ms": 0.731168,
       "reduction_ctas": 3,
       "iterations": 8,
       "correctness_gate": "known_solution_plus_fp32_true_residual",
@@ -38,8 +38,8 @@
       "mode": "cooperative_grid:end_to_end",
       "selected_route": "cooperative_grid",
       "timing_domain": "end_to_end",
-      "median_ms": 2.752441,
-      "max_latency_ms": 6.881103,
+      "median_ms": 2.642597,
+      "max_latency_ms": 6.606492,
       "reduction_ctas": 5,
       "iterations": 8,
       "correctness_gate": "known_solution_plus_fp32_true_residual",
@@ -52,8 +52,8 @@
       "mode": "cooperative_grid:device_event",
       "selected_route": "cooperative_grid",
       "timing_domain": "device_event",
-      "median_ms": 0.513587,
-      "max_latency_ms": 1.283968,
+      "median_ms": 0.538221,
+      "max_latency_ms": 1.345552,
       "reduction_ctas": 5,
       "iterations": 8,
       "correctness_gate": "known_solution_plus_fp32_true_residual",
@@ -66,8 +66,8 @@
       "mode": "cooperative_grid:end_to_end",
       "selected_route": "cooperative_grid",
       "timing_domain": "end_to_end",
-      "median_ms": 6.632047,
-      "max_latency_ms": 16.580118,
+      "median_ms": 7.182736,
+      "max_latency_ms": 17.95684,
       "reduction_ctas": 9,
       "iterations": 8,
       "correctness_gate": "known_solution_plus_fp32_true_residual",
@@ -80,8 +80,8 @@
       "mode": "cooperative_grid:device_event",
       "selected_route": "cooperative_grid",
       "timing_domain": "device_event",
-      "median_ms": 1.03271,
-      "max_latency_ms": 2.581776,
+      "median_ms": 1.015552,
+      "max_latency_ms": 2.53888,
       "reduction_ctas": 9,
       "iterations": 8,
       "correctness_gate": "known_solution_plus_fp32_true_residual",
@@ -94,8 +94,8 @@
       "mode": "cooperative_grid:end_to_end",
       "selected_route": "cooperative_grid",
       "timing_domain": "end_to_end",
-      "median_ms": 1.587581,
-      "max_latency_ms": 3.968952,
+      "median_ms": 1.60096,
+      "max_latency_ms": 4.0024,
       "reduction_ctas": 3,
       "iterations": 6,
       "correctness_gate": "known_solution_plus_fp32_true_residual",
@@ -108,8 +108,8 @@
       "mode": "cooperative_grid:device_event",
       "selected_route": "cooperative_grid",
       "timing_domain": "device_event",
-      "median_ms": 0.367206,
-      "max_latency_ms": 0.918016,
+      "median_ms": 0.296947,
+      "max_latency_ms": 0.742368,
       "reduction_ctas": 3,
       "iterations": 6,
       "correctness_gate": "known_solution_plus_fp32_true_residual",
@@ -122,8 +122,8 @@
       "mode": "cooperative_grid:end_to_end",
       "selected_route": "cooperative_grid",
       "timing_domain": "end_to_end",
-      "median_ms": 2.693219,
-      "max_latency_ms": 6.733047,
+      "median_ms": 2.542125,
+      "max_latency_ms": 6.355313,
       "reduction_ctas": 5,
       "iterations": 6,
       "correctness_gate": "known_solution_plus_fp32_true_residual",
@@ -136,8 +136,8 @@
       "mode": "cooperative_grid:device_event",
       "selected_route": "cooperative_grid",
       "timing_domain": "device_event",
-      "median_ms": 0.572154,
-      "max_latency_ms": 1.430384,
+      "median_ms": 0.394662,
+      "max_latency_ms": 0.986656,
       "reduction_ctas": 5,
       "iterations": 6,
       "correctness_gate": "known_solution_plus_fp32_true_residual",
@@ -150,8 +150,8 @@
       "mode": "cooperative_grid:end_to_end",
       "selected_route": "cooperative_grid",
       "timing_domain": "end_to_end",
-      "median_ms": 6.833705,
-      "max_latency_ms": 17.084263,
+      "median_ms": 7.147036,
+      "max_latency_ms": 17.86759,
       "reduction_ctas": 9,
       "iterations": 6,
       "correctness_gate": "known_solution_plus_fp32_true_residual",
@@ -164,8 +164,8 @@
       "mode": "cooperative_grid:device_event",
       "selected_route": "cooperative_grid",
       "timing_domain": "device_event",
-      "median_ms": 0.974483,
-      "max_latency_ms": 2.436208,
+      "median_ms": 0.611917,
+      "max_latency_ms": 1.529792,
       "reduction_ctas": 9,
       "iterations": 6,
       "correctness_gate": "known_solution_plus_fp32_true_residual",

benchmarks/rocm/benchmark_rocm_adafactor.py

@@ -29,7 +29,7 @@ def artifact() -> rt.RuntimeArtifact:
             "op_name": "tessera.adafactor",
             "result": "output",
             "operands": ["parameter", "gradient", "row", "col"],
-            "kwargs": {"lr": 1e-2, "beta2": 0.9, "eps": 1e-6},
+            "kwargs": {"lr": 1e-2, "beta2": 0.9, "eps": 1e-6, "step": 8},
         }],
     })
 
@@ -47,7 +47,7 @@ def backward_artifact() -> rt.RuntimeArtifact:
             "result": "gradients",
             "operands": ["parameter", "gradient", "row", "col"],
             "out_cotangent": "dy",
-            "kwargs": {"lr": 1e-2, "beta2": 0.9, "eps": 1e-6},
+            "kwargs": {"lr": 1e-2, "beta2": 0.9, "eps": 1e-6, "step": 8},
         }],
     })
 

docs/audit/backend/apple/todo.md

@@ -7,6 +7,32 @@ last_updated: 2026-08-29
 ---
 
 # Apple compiler, exact-device, and performance plan
+Cross-backend sync `ADAFACTOR-BIAS-CORRECTION-2026-08-30` — **shared numerical
+policy changed; per-backend outcome below.**
+`optim.adafactor_decay` makes the Adafactor second-moment decay step-dependent
+(`b2_t = b2*(1 - b2^(t-1))/(1 - b2^t)`), removing an early-step update
+inflation of 1/sqrt(1 - b2^t) — 31.6x at step 1, 10.0x at step 10, 1.26x at
+step 1000 for the default beta2. The correction is applied HOST-SIDE as a
+scalar decay, so **no kernel ABI moves**: every physical kernel already takes
+`beta2` as a scalar and receives the effective value instead of the nominal
+one. The flat op gained an optional `step` kwarg matching the `adam`/`adamw`
+ABI beside it.
+
+Two contract details a backend owner needs to know. `state["v"]` now carries
+the DEBIASED estimate rather than the raw EMA, so the state dict grew a
+`v_representation` marker and a state without one is migrated on load rather
+than misread. And an absent `step` is NOT treated as step 1 — `decay(b2, 1)`
+is exactly 0, so defaulting would have made a stateful caller that never
+passes one discard its own moments; such a caller keeps the legacy
+uncorrected decay.
+
+*Apple outcome: not applicable.* The Apple backend exposes no Adafactor
+kernel — neither the Accelerate CPU lane nor the MSL/MPS GPU lane registers
+one, so there is no Apple code path this policy reaches. The reference
+`optim.adafactor` runs as host numpy here like any other pure-Python lane and
+is covered by `tests/unit/test_s10_optim.py`; nothing Apple-specific to
+validate.
+
 Cross-backend sync `P2-REVIEW-SHARED-PASSES-2026-08-29` — **15 shared MLIR
 passes changed; only the Mac's fixture set could be run.**
 The P2 code-review batch touched passes every backend lowers through:

docs/audit/backend/nvidia/todo.md

@@ -7,6 +7,57 @@ last_updated: 2026-08-29
 ---
 
 # NVIDIA compiler test-suite evaluation and rearchitecture
+Cross-backend sync `ADAFACTOR-BIAS-CORRECTION-2026-08-30` — **shared numerical
+policy changed; per-backend outcome below.**
+`optim.adafactor_decay` makes the Adafactor second-moment decay step-dependent
+(`b2_t = b2*(1 - b2^(t-1))/(1 - b2^t)`), removing an early-step update
+inflation of 1/sqrt(1 - b2^t) — 31.6x at step 1, 10.0x at step 10, 1.26x at
+step 1000 for the default beta2. The correction is applied HOST-SIDE as a
+scalar decay, so **no kernel ABI moves**: every physical kernel already takes
+`beta2` as a scalar and receives the effective value instead of the nominal
+one. The flat op gained an optional `step` kwarg matching the `adam`/`adamw`
+ABI beside it.
+
+Two contract details a backend owner needs to know. `state["v"]` now carries
+the DEBIASED estimate rather than the raw EMA, so the state dict grew a
+`v_representation` marker and a state without one is migrated on load rather
+than misread. And an absent `step` is NOT treated as step 1 — `decay(b2, 1)`
+is exactly 0, so defaulting would have made a stateful caller that never
+passes one discard its own moments; such a caller keeps the legacy
+uncorrected decay.
+
+*NVIDIA outcome: follow-up required.* `sm120_adafactor_*` receives the
+effective decay through the existing scalar; `tests/device/nvidia/test_optimizer_reverse.py`
+was migrated to pass `step`. Not executed on sm_120 in this batch — an exact-device
+run of the optimizer lanes is owed.
+
+Cross-backend sync `P3-DEVICE-VERIFIED-2026-08-30` — **the two NVIDIA rows
+owed by `P3-SOURCE-ONLY` are now measured, and one of them was a regression.**
+
+* `emit/nvidia_solver_krylov.py` `tsr_matvec` — the warp-per-row rewrite was
+  shipped on a reasoned access-pattern claim. Measured on an RTX 5070
+  (sm_120), medians of 9 reps, device_event: **dense_cg 0.44-0.63x (a
+  REGRESSION of up to 2.3x)** and **dense_gmres 1.22-1.56x (a win)**. The
+  coalescing argument was correct and still lost, because a COOPERATIVE
+  launch caps the grid at what stays resident, so warp-per-row also buys 32x
+  fewer rows in flight. The solvers no longer share one matvec:
+  `tsr_matvec_scalar` for CG, `tsr_matvec_warp` for GMRES, with the table in
+  the source. Re-measured after the split: CG back to 1.00-1.06x of scalar,
+  GMRES keeps 1.24-1.56x. `benchmarks/baselines/nvidia_sm120_solver_krylov_performance.json`
+  was recorded with the OLD matvec and **passed throughout the regression** —
+  re-recorded at 15 reps / 5 warmup, and the ratchet now measures reality.
+* `emit/nvidia_cuda.py` flash-backward cleanup — the 20 Krylov/solver device
+  tests and the flash-backward route tests pass on sm_120. An induced
+  allocation failure is still not exercised; that remains the honest gap.
+
+Also closed here: the `rc=5` invoke failure (the runtime dispatches scheduled
+sm_120 matmuls by NAME PREFIX while the compiler named the kernel after the
+caller's Graph function) and the sm_120 packager reading matmul epilogue
+edges from `op.kwargs` when the verifier requires operands. Device suite:
+**81 failed -> 5 failed / 844 passed.** The 5 remaining are 4 stale
+shared-staging assertions (`__tessera_sm120_ab_stage_bf16`, pre-existing and
+a routing question, not a test-editing one) and NCCL not being installed.
+
 Cross-backend sync `P3-SOURCE-ONLY-2026-08-30` — **two rows are fixed in
 source and have never run on a GPU; they are this queue's to close.**
 The P3 batch changed two NVIDIA emitters with no CUDA host available:

docs/audit/backend/rocm/todo.md

@@ -6,6 +6,47 @@ scope: ROCm backend implementation and exact-device proof
 ---
 
 # ROCm backend TODO
+Cross-backend sync `ADAFACTOR-BIAS-CORRECTION-2026-08-30` — **shared numerical
+policy changed; per-backend outcome below.**
+`optim.adafactor_decay` makes the Adafactor second-moment decay step-dependent
+(`b2_t = b2*(1 - b2^(t-1))/(1 - b2^t)`), removing an early-step update
+inflation of 1/sqrt(1 - b2^t) — 31.6x at step 1, 10.0x at step 10, 1.26x at
+step 1000 for the default beta2. The correction is applied HOST-SIDE as a
+scalar decay, so **no kernel ABI moves**: every physical kernel already takes
+`beta2` as a scalar and receives the effective value instead of the nominal
+one. The flat op gained an optional `step` kwarg matching the `adam`/`adamw`
+ABI beside it.
+
+Two contract details a backend owner needs to know. `state["v"]` now carries
+the DEBIASED estimate rather than the raw EMA, so the state dict grew a
+`v_representation` marker and a state without one is migrated on load rather
+than misread. And an absent `step` is NOT treated as step 1 — `decay(b2, 1)`
+is exactly 0, so defaulting would have made a stateful caller that never
+passes one discard its own moments; such a caller keeps the legacy
+uncorrected decay.
+
+*ROCm outcome: follow-up required.* The gfx1151 `adafactor_row|col|mean|update`
+kernels take `beta2` as a scalar and are unchanged; `tests/unit/test_rocm_optimizer_compiled.py`
+and `benchmarks/rocm/benchmark_rocm_adafactor.py` were migrated to pass `step`.
+The host-free lanes pass here, but a gfx1151 optimizer execution comparing the
+corrected trajectory against the numpy reference is owed.
+
+Cross-backend sync `P3-DEVICE-VERIFIED-2026-08-30` — **the batch FFT seam
+written blind against the `.hip` signature is real, correct and faster.**
+`RocmStockhamFFTCandidate.run_rows` was landed unexecuted, declining to the
+per-row path when the image lacked the batch ABI — which is precisely the
+state it was tested in. Measured here on gfx1151: the shipped image DOES
+export `ts_fft_stockham_amd_hostptr_batch`, the candidate reports available,
+and a (512, 256) transform runs on lane `rocm_stockham` with **max abs error
+1.9e-05** against `numpy.fft` and **62.6 ms vs 237.7 ms per-row — 3.80x**.
+
+Also verified here: the rank-4 dropout `stream_offset` contract change. The
+fixture that broke the PREVIOUS attempt at this row,
+`phase3/streaming_attention_backward_rocm.mlir`, is `REQUIRES:
+tessera-rocm-backend` and therefore unsupported on the Mac — it runs on this
+box and passes. `lit` 439 discovered / 384 passed / 0 failed;
+`check-tessera-rocm` 67 passed / 1 unsupported, identical to baseline.
+
 Cross-backend sync `P3-SOURCE-ONLY-2026-08-30` — **one row is written from
 the `.hip` signature and has never executed.**
 The P3 batch added `RocmStockhamFFTCandidate.run_rows`, wired to

docs/audit/backend/x86/todo.md

@@ -8,6 +8,30 @@ scope: x86 AVX-512 implementation/proof and AMX access planning
 ---
 
 # x86 backend TODO
+Cross-backend sync `ADAFACTOR-BIAS-CORRECTION-2026-08-30` — **shared numerical
+policy changed; per-backend outcome below.**
+`optim.adafactor_decay` makes the Adafactor second-moment decay step-dependent
+(`b2_t = b2*(1 - b2^(t-1))/(1 - b2^t)`), removing an early-step update
+inflation of 1/sqrt(1 - b2^t) — 31.6x at step 1, 10.0x at step 10, 1.26x at
+step 1000 for the default beta2. The correction is applied HOST-SIDE as a
+scalar decay, so **no kernel ABI moves**: every physical kernel already takes
+`beta2` as a scalar and receives the effective value instead of the nominal
+one. The flat op gained an optional `step` kwarg matching the `adam`/`adamw`
+ABI beside it.
+
+Two contract details a backend owner needs to know. `state["v"]` now carries
+the DEBIASED estimate rather than the raw EMA, so the state dict grew a
+`v_representation` marker and a state without one is migrated on load rather
+than misread. And an absent `step` is NOT treated as step 1 — `decay(b2, 1)`
+is exactly 0, so defaulting would have made a stateful caller that never
+passes one discard its own moments; such a caller keeps the legacy
+uncorrected decay.
+
+*x86 outcome: follow-up required.* `tessera_x86_avx512_adafactor_*` takes
+`beta2` as a scalar and is unchanged; `tests/unit/test_x86_optimizer_compiled.py`
+was migrated to pass `step`. AVX-512 execution lives on the Strix Halo box, so
+an exact-device run of the corrected optimizer is owed there.
+
 Cross-backend sync `P2-REVIEW-SHARED-PASSES-2026-08-29` — **15 shared MLIR
 passes changed; only the Mac's fixture set could be run.**
 The P2 code-review batch touched passes every backend lowers through: