[Bugfix] Use FP32 state and E4M3 defaults for DFlash2

README.md

@@ -47,6 +47,13 @@ Demo: [4× V100 running Qwen3.8-27B-NVFP4-DFlash2](https://www.bilibili.com/vide
 
 # 📊 Performance First
 
+SM70 Flash-V100 now resolves `--kv-cache-dtype fp8` to E4M3. DFlash2 E4M3
+verification uses repaired FP32 attention state, and the Qwen3.8 DFlash2
+configuration enables FP32 logits by default. Rebuild Flash-V100 for precision
+revision 4; see [the precision contract and validation](docs/design/sm70_dflash2_fp32_defaults.md).
+Historical E5M2/FP16-partial performance results below keep their original
+configuration and are not speed claims for these precision defaults.
+
 ## Long-Context Attention: 17.92 → 47.1 → ≈60.8 TFLOP/s
 
 | Stage | Evidence | Useful causal Attention compute | Notes |

benchmarks/benchmark_sm70_dflash2_fp32_attention.py

@@ -0,0 +1,143 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Compare legacy E4M3 partials with repaired FP32 attention, not FP8 loss."""
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+
+import torch
+from flash_attn_v100 import (
+    flash_attn_grouped_e4m3_fp32_paged,
+    flash_attn_grouped_verify_paged,
+)
+from flash_attn_v100.flash_attn_interface import flash_attn_v100_cuda
+
+
+def measure(length, page, samples):
+    torch.manual_seed(20260908)
+    pages = (length + page - 1) // page
+    capacity = pages * page
+    q = torch.randn((8, 6, 256), dtype=torch.float16, device="cuda")
+    raw = torch.randn((2, capacity, 1, 256), dtype=torch.float16, device="cuda")
+    encoded = raw.to(torch.float8_e4m3fn).view(torch.uint8)
+    backing = torch.empty((pages, 2, page, 1, 256), dtype=torch.uint8, device="cuda")
+    k, v = backing.unbind(1)
+    order = torch.randperm(pages, device="cuda")
+    k[order] = encoded[0].reshape_as(k)
+    v[order] = encoded[1].reshape_as(v)
+    table = order.int()[None].contiguous()
+    seq = torch.tensor([length], dtype=torch.int32, device="cuda")
+    rows = torch.arange(length - 7, length + 1, dtype=torch.int32, device="cuda")
+    outputs = [torch.empty_like(q), torch.empty_like(q)]
+
+    def legacy():
+        flash_attn_grouped_verify_paged(
+            q,
+            k,
+            v,
+            table,
+            seq,
+            out=outputs[0],
+            softmax_scale=0.0625,
+            kv_cache_dtype="fp8_e4m3",
+            k_scale=0.5,
+            v_scale=1.25,
+            one_pass=True,
+        )
+
+    def repaired():
+        flash_attn_grouped_e4m3_fp32_paged(
+            q,
+            k,
+            v,
+            table,
+            rows,
+            out=outputs[1],
+            softmax_scale=0.0625,
+            k_scale=0.5,
+            v_scale=1.25,
+        )
+
+    graphs = []
+    for call in (legacy, repaired):
+        for _ in range(20):
+            call()
+        graph = torch.cuda.CUDAGraph()
+        with torch.cuda.graph(graph):
+            call()
+        graphs.append(graph)
+    timings = [[], []]
+    for _ in range(samples):
+        for index in (0, 1, 1, 0):
+            start = torch.cuda.Event(enable_timing=True)
+            end = torch.cuda.Event(enable_timing=True)
+            start.record()
+            graphs[index].replay()
+            end.record()
+            end.synchronize()
+            timings[index].append(start.elapsed_time(end))
+
+    # FP64 resolves the FP16 output-rounding floor; also report the requested
+    # PyTorch FP32 oracle on exactly the same quantized KV and causal rows.
+    errors = [{}, {}]
+    for dtype in (torch.float32, torch.float64):
+        rk = encoded[0, :length, 0].view(torch.float8_e4m3fn).to(dtype) * 0.5
+        rv = encoded[1, :length, 0].view(torch.float8_e4m3fn).to(dtype) * 1.25
+        scores = q.transpose(0, 1).to(dtype) @ rk.T * 0.0625
+        mask = torch.arange(length, device="cuda")[None] >= rows[:, None]
+        scores.masked_fill_(mask[None], -torch.inf)
+        expected = (scores.softmax(-1) @ rv).transpose(0, 1)
+        denominator = expected.norm()
+        floor = float((expected.half().to(dtype) - expected).norm() / denominator)
+        for error, output in zip(errors, outputs):
+            diff = output.to(dtype) - expected
+            error[str(dtype)] = {
+                "relative_l2": float(diff.norm() / denominator),
+                "max_abs": float(diff.abs().max()),
+                "fp16_rounding_floor": floor,
+                "finite": bool(torch.isfinite(output).all()),
+            }
+    result = {"length": length, "page": page, "variants": {}}
+    for name, times, error in zip(("legacy_half", "repaired_fp32"), timings, errors):
+        values = torch.tensor(times)
+        result["variants"][name] = {
+            "median_ms": float(values.median()),
+            "p10_ms": float(values.quantile(0.1)),
+            "p90_ms": float(values.quantile(0.9)),
+            "samples_ms": times,
+            "error": error,
+        }
+    return result
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--samples", type=int, default=25)
+    args = parser.parse_args()
+    if args.output.exists():
+        raise FileExistsError(args.output)
+    if torch.cuda.get_device_capability() != (7, 0):
+        raise RuntimeError("requires an idle SM70 GPU")
+    native = Path(flash_attn_v100_cuda.__file__).resolve()
+    result = {
+        "torch": torch.__version__,
+        "cuda": torch.version.cuda,
+        "device": torch.cuda.get_device_name(),
+        "native_path": str(native),
+        "native_sha256": hashlib.sha256(native.read_bytes()).hexdigest(),
+        "precision_version": flash_attn_v100_cuda.grouped_e4m3_fp32_precision_version(),
+        "contract": "B1/q8/H6/Hkv1/D256, same E4M3 KV, graph ABBA; not E2E",
+        "cases": [],
+    }
+    for length in (8192, 65536, 131072, 262144):
+        row = measure(length, 3296, args.samples)
+        result["cases"].append(row)
+        print(json.dumps(row), flush=True)
+        args.output.write_text(json.dumps(result, indent=2) + "\n")
+
+
+if __name__ == "__main__":
+    main()

docs/design/sm70_dflash2_fp32_defaults.md

@@ -0,0 +1,109 @@
+# DFlash2 E4M3 attention with FP32 state
+
+The SM70 Flash-V100 `fp8` KV alias now resolves to `fp8_e4m3`. Explicit
+`fp8_e5m2` remains available for reproducing older deployments. Explicit
+formats, checkpoint-resolved formats and non-SM70 backends are preserved.
+This changes the FP8 alias, not the general `auto` cache policy or model weights.
+
+For E4M3 DFlash2 target verification, the backend no longer selects the legacy
+grouped entry that stores normalized attention partials in FP16. Its advertised
+E4M3 support only proves byte-format compatibility. The backend routes compatible
+single-request q2–8/H6/Hkv1/D256 input through the repaired FP32 entry, using
+the metadata builder's live per-query lengths, including zero padding.
+
+The repaired computation retains compensated QK accumulation, compensated
+probabilities, tile-local FP32 PV accumulation, FP32 unnormalized partition
+numerators, separate FP32 max/sum, and FP32 combination. KV storage remains
+E4M3; Tensor Core operands and the final activation remain FP16. This is not a
+full-FP32 model and does not remove FP8 quantization error.
+
+Precision capability revision **4** adds page sizes **1728/3456** to the existing
+800/848/1616/1648/3296 runtime-stride implementation. The arithmetic is unchanged
+from repaired revision 3. It also adds FP32 scalar E4M3 partial storage and
+reduction. The wrapper requires revision 4 so a new page or FP32 workspace
+cannot reach an incompatible native binary. Rebuild the extension and restart
+workers together with this Python update.
+
+Unsupported grouped shapes, independent-request batches, q16 and explicit grouped
+rollback use the scalar path with FP32 accumulation and the new FP32 split
+workspace. Scalar and XQA workspaces are separated by partial dtype in the cache.
+They do not re-enter the legacy E4M3 FP16-partial verifier. If the native binary
+lacks revision 4, E4M3 scalar calls raise a rebuild error instead of silently
+storing half partials. A DFlash2 target q1 also uses FP32 scalar state instead of
+the half-partial XQA wave route. Ordinary non-DFlash XQA dispatch is unchanged.
+The low-level legacy operator remains available for numerical A/B tests and
+explicit E5M2 compatibility; it is not the E4M3 serving policy.
+
+The Qwen3.8 DFlash2 configuration also enables
+`VLLM_SM70_DFLASH2_FP32_LOGITS=1` by default so candidate rerank and dense fallback
+retain FP32 logits. This is model-scoped configuration; the global environment
+default remains off for unrelated models. Explicit environment overrides are
+preserved. This does not enable MTP or change the sampling distribution settings.
+
+## Observability and reproduction
+
+A compatible worker reports `E4M3 grouped FP32 route selected` with its page,
+query rows and FP32 state representation. Route summaries include
+`prefill_smallq_e4m3_grouped_fp32` and `fp8_kv_decode_grouped_fp32`.
+Check these alongside the resolved KV dtype and FP32-logit preparation; an image
+name or requested flag is insufficient evidence of the numerical route.
+
+With an isolated Python 3.12 runtime, CUDA 12.8, Torch 2.10+cu128 and idle V100s:
+
+```bash
+CUDA_VISIBLE_DEVICES=1 .venv/bin/python -m pytest -q \
+  tests/v1/attention/test_sm70_flash_v100_policy.py \
+  tests/v1/attention/test_sm70_e4m3_grouped.py \
+  tests/v1/spec_decode/test_dflash2.py \
+  tests/kernels/attention/test_sm70_grouped_e4m3_fp32.py
+
+CUDA_VISIBLE_DEVICES=1 .venv/bin/python \
+  benchmarks/benchmark_sm70_dflash2_fp32_attention.py --output operator.json
+```
+
+Point `PYTHONPATH` at the task's Python package and rebuilt extension. The
+operator benchmark holds E4M3 bytes, query, causal visibility, KV scales and
+native library fixed. It reports 20 warmups, 25 ABBA blocks (50 samples per
+variant), latency percentiles, PyTorch FP32/FP64 arithmetic error and the FP16
+output-rounding floor. It separates avoidable attention error from cache
+quantization, and is not a model-throughput benchmark.
+
+Model acceptance must be checked with request-level draft/accepted-token
+counter deltas. A rolling logger's mean acceptance length does not provide a
+matched request comparison. FP32 is the chosen arithmetic contract, but lower
+operator L2 alone does not establish model-quality improvement or guarantee
+identical sampled tokens.
+
+## Rebuilt operator results, 2026-09-08
+
+The initial routing build passed **420 tests** before the scalar fallback was
+extended. The native q8 tests include 8K/64K/128K/256K, newly admitted
+1728/3456 pages, relocated pages, non-unit scales and CUDA Graph replay with
+changed/zero row lengths. Numerical assertions bound error relative to the
+FP16 output-rounding floor, rather than only using an aggregate loose tolerance.
+The final scalar/q1 policy follow-up passes **338 checks**, including all five
+new scalar tests. The separate native/planner run passed 102 checks and exposed
+an incorrect test assertion that eager and graph streams must share a workspace;
+the corrected tests check reuse within one stream and FP32 buffers in both.
+This was a test expectation error, not an attention numerical failure.
+
+Physical GPU2, V100-SXM2-32GB, Torch 2.10.0+cu128, CUDA 12.8.93/GCC12,
+q8/page3296, one rebuilt library and paired graph samples:
+
+| Context | Legacy ms | FP32 ms | Legacy relative L2 | FP32 relative L2 |
+| --- | ---: | ---: | ---: | ---: |
+| 8192 | 0.0778 | 0.1034 | 0.00035316 | 0.00020792 |
+| 65536 | 0.2714 | 0.4209 | 0.00034410 | 0.00020785 |
+| 131072 | 0.4977 | 0.7916 | 0.00035442 | 0.00020903 |
+| 262144 | 0.9462 | 1.5206 | 0.00033226 | 0.00020327 |
+
+L2 uses PyTorch FP32 attention over identical quantized KV. The companion FP64
+reference gives an FP16 rounding floor of 0.000203267 at 256K. The operator
+has about 39% lower L2 and 61% higher latency at that length; this is a deliberate
+precision choice, not a speedup. It does not establish a model acceptance gain.
+No cross-GPU absolute timing is combined.
+
+Native library SHA256:
+`d2f70b502985af14fe816b379ffa319ca4887191dfab311d62836ee121a41ef3`.
+Raw artifacts are indexed under `dflash2-e4m3-fp32-default-20260908`, including
+`operator-r2.json` with all samples and both numerical references.

docs/design/sm70_e4m3_grouped_fp32.md

@@ -1,5 +1,11 @@
 # Experimental E4M3 grouped attention with FP32 partial state
 
+**2026-09-08 policy update:** [DFlash2 FP32 defaults](sm70_dflash2_fp32_defaults.md)
+supersede the default-routing and KV-alias decisions recorded below. E4M3
+DFlash2 verification now uses repaired FP32 state; revision 4 adds the
+1728/3456 page layouts. Earlier measurements and failed model gates retain
+their original artifact attribution.
+
 ## Scope and admission
 
 **Current mainline audit decision (2026-09-07): enabled for compatible

docs/design/sm70_glm53_flash_nvfp4.md

@@ -114,7 +114,9 @@ The adaptation is divided into independently testable surfaces:
    workspace before two Tensor Core GEMMs. The older direct scalar kernel is
    retained only as a reference/test path and is not accepted for B1 decode.
    Use the explicit `fp8_e4m3` cache dtype because the historical generic SM70
-   `fp8` alias resolves to E5M2 for other model families.
+   `fp8` alias historically resolved to E5M2 for other model families. The
+   [E4M3 default update](sm70_dflash2_fp32_defaults.md) changes that alias;
+   explicit `fp8_e4m3` keeps this recipe independent of that version boundary.
 9. Keep all GLM mHC4/H4096 execution on native SM70 kernels. Small-M fused
    decode follows the DeepSeek-V4 FP32 staging design, but its final Sinkhorn,
    residual mix, and RMSNorm stage is a dedicated single-CTA CUDA kernel for

docs/design/sm70_v100_migration_control.md

@@ -2,6 +2,19 @@
 
 Date: 2026-05-30
 
+## DFlash2 E4M3 FP32 default policy, 2026-09-08
+
+[The precision-default change](sm70_dflash2_fp32_defaults.md) is based on
+`56f534e672657a6c7599afd6c0dcb2e2c211b2e3`. It removes E4M3 admission to the
+legacy FP16-partial verifier, admits DFlash2 pages 1728/3456 to repaired FP32
+attention, resolves the SM70 `fp8` alias to E4M3, and enables FP32 logits in
+the existing Qwen3.8 DFlash2 default configuration. Explicit E5M2 remains
+available. No MTP enablement is part of this change.
+
+The legacy half-partial microbenchmark is retained for the speed/precision
+comparison; its higher speed is not a reason to restore it to E4M3 serving.
+Only measured model results may establish acceptance or throughput effects.
+
 ## v37 prefill integration: model-parity hold, 2026-09-07
 
 Draft [PR548](https://github.com/1CatAI/1Cat-vLLM/pull/548) integrates the

flash-attention-v100/flash_attn_v100/flash_attn_interface.py

@@ -124,11 +124,12 @@ def _allocate_decode_workspace(
     num_heads: int,
     head_dim: int,
     max_num_partitions: int,
+    partial_dtype: torch.dtype = torch.float16,
 ) -> _DecodeWorkspace:
     return _DecodeWorkspace(
         tmp_out=torch.empty(
             (batch_capacity, num_heads, max_num_partitions, head_dim),
-            dtype=torch.float16,
+            dtype=partial_dtype,
             device=q.device,
         ),
         max_logits=torch.empty(
@@ -313,6 +314,7 @@ def _get_decode_workspace_for_plan(
     head_dim: int,
     plan: _DecodePlan,
     active_num_partitions: torch.Tensor | None = None,
+    partial_dtype: torch.dtype = torch.float16,
 ):
     device_index = q.device.index if q.device.index is not None else -1
     stream_id = _workspace_stream_id(q.device)
@@ -323,6 +325,7 @@ def _get_decode_workspace_for_plan(
         num_heads,
         head_dim,
         plan.partition_size,
+        partial_dtype,
     )
 
     workspace = _decode_workspace_cache.get(key) if _can_cache_workspace(q) else None
@@ -338,6 +341,7 @@ def _get_decode_workspace_for_plan(
             max_num_partitions=_round_decode_partition_capacity(
                 plan.workspace_num_partitions
             ),
+            partial_dtype=partial_dtype,
         )
         if _can_cache_workspace(q):
             _decode_workspace_cache[key] = workspace
@@ -967,6 +971,11 @@ def flash_attn_decode_paged(
     if softmax_scale is None:
         softmax_scale = q.shape[-1] ** -0.5
 
+    e4m3_fp32 = kv_cache_dtype in ("fp8", "fp8_e4m3")
+    if e4m3_fp32 and not flash_attn_grouped_e4m3_fp32_available():
+        raise RuntimeError(
+            "Rebuild Flash-V100 for E4M3 FP32 scalar decode (precision revision 4)"
+        )
     q = maybe_contiguous(q)
     block_table = maybe_contiguous(block_table)
     seq_lens = maybe_contiguous(seq_lens)
@@ -1011,6 +1020,7 @@ def flash_attn_decode_paged(
             head_dim=head_dim,
             plan=plan,
             active_num_partitions=active_num_partitions,
+            partial_dtype=torch.float32 if e4m3_fp32 else torch.float16,
         )
     )
 
@@ -1078,7 +1088,7 @@ def flash_attn_grouped_e4m3_fp32_available() -> bool:
     return (
         hasattr(flash_attn_v100_cuda, "grouped_e4m3_fp32_paged_fwd")
         and callable(version)
-        and int(version()) >= 3
+        and int(version()) >= 4
     )
 
 
@@ -1094,7 +1104,7 @@ def flash_attn_grouped_e4m3_fp32_paged(
     k_scale: float = 1.0,
     v_scale: float = 1.0,
 ) -> torch.Tensor:
-    """Experimental E4M3 q2..8/GQA6/D256 attention over one KV sequence.
+    """E4M3 q2..8/GQA6/D256 attention over one KV sequence.
 
     Row lengths are authoritative GPU metadata, not inferred from padded Q.
     Zero lengths produce zero outputs. All positive lengths must fit the
@@ -1103,10 +1113,11 @@ def flash_attn_grouped_e4m3_fp32_paged(
     Tensor Core operands and final output remain FP16. KV must encode E4M3.
     Precision revision 3 retains FP32 numerators and separate max/sum until
     the final normalization, as well as compensated QK/P and tile-local PV.
+    Revision 4 adds DFlash2 1728/3456 pages and FP32 scalar fallback workspace.
     """
     if not flash_attn_grouped_e4m3_fp32_available():
         raise RuntimeError(
-            "Rebuild Flash-V100 for E4M3 grouped FP32 precision revision 3"
+            "Rebuild Flash-V100 for E4M3 grouped FP32 precision revision 4"
         )
     workspace = _get_grouped_verify_workspace(q, partial_dtype=torch.float32)
     return flash_attn_v100_cuda.grouped_e4m3_fp32_paged_fwd(