From 99bf3f13f13012b72d6d67e49572d686b45b2833 Mon Sep 17 00:00:00 2001 From: MadeBy561 Date: Mon, 20 Jul 2026 15:02:44 -0400 Subject: [PATCH] glm52: calibrated NVFP4 MLA KV outer scales + serve wiring Per-layer outer-scale calibration for nvfp4_ds_mla KV (VLLM_NVFP4_MLA_SCALES_FILE, format v1) with an explicit default-off knob in serve-glm52.sh. Collapses the NVFP4-vs-FP8 KV KLD gap to ~+0.008-0.009 (0.1345/0.1356 vs 0.1263, 5 fresh boots each, rtx6kpro protocol) while raising max context from 373k to 550k/600k+ on 4x96GB. Co-Authored-By: Claude Fable 5 --- kv-scales/README.md | 49 +++++ ...-nvfp4-nf3-hybrid_mla_outer_scales_v1.json | 178 ++++++++++++++++++ serve-glm52.sh | 2 + 3 files changed, 229 insertions(+) create mode 100644 kv-scales/README.md create mode 100644 kv-scales/glm52-nvfp4-nf3-hybrid_mla_outer_scales_v1.json diff --git a/kv-scales/README.md b/kv-scales/README.md new file mode 100644 index 000000000000..36b37d5aa188 --- /dev/null +++ b/kv-scales/README.md @@ -0,0 +1,49 @@ +# GLM-5.2 NVFP4 MLA KV outer scales + +Per-layer calibration for the `nvfp4_ds_mla` KV cache writer +(`VLLM_NVFP4_MLA_SCALES_FILE`, format `nvfp4_ds_mla_outer_scale_v1`). + +GLM-5.2's post-RMSNorm 512-dim `kv_c` latent spans a ~240x amplitude range +across layers. With the default outer scale of 1.0, shallow layers quantize +with E4M3 block scales at or below the subnormal floor, and shallow-layer KV +error is strongly amplified downstream. `s_l = max_abs(kv_c_normed) / (6*448)` +re-centers every layer so its largest block scale lands at the top of the +E4M3 range. + +## Files + +- `glm52-nvfp4-nf3-hybrid_mla_outer_scales_v1.json` — calibrated on + `madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid` (K64R16), Salesforce/wikitext + (wikitext-2-raw-v1, test), 2048-token context, TP4; per-layer envelope with + an independent community capture of the same base model for shallow-layer + headroom. `max_abs` per layer is included for auditability. + +## Usage + +```bash +VLLM_NVFP4_MLA_SCALES_FILE=/path/to/glm52-nvfp4-nf3-hybrid_mla_outer_scales_v1.json \ + ./serve-glm52.sh # nvfp4_ds_mla + B12X_MLA_SPARSE only; inert otherwise +``` + +## Results (teacher-forced prefill KLD vs BF16 reference, 5 fresh boots each) + +| KV config | mean +/- sd | max ctx (4x96GB) | +|---|---|---| +| fp8_ds_mla | 0.1263 +/- 0.0030 | 373k | +| nvfp4_ds_mla + scales, bf16 rope | 0.1345 +/- 0.0035 | 550k | +| nvfp4_ds_mla + scales, fp8 rope (`KV_FP8_ROPE=1`) | 0.1356 +/- 0.0054 | 600k+ | +| nvfp4_ds_mla, no scales, bf16 rope | 0.158 | 550k | +| nvfp4_ds_mla, no scales, fp8 rope | 0.168 | 600k+ | + +Protocol: local-inference-lab/rtx6kpro `benchmarks/glm52-kld-evaluation.md` +(festr2 2026-07-08 reference logits, one fixed 2048-token window, 2047 +positions, full 154,880 vocab, `KL(ref || candidate)`). + +## Cache invalidation (required) + +CuTeDSL folds the outer-scale multiply out of the kernel when `latent_scale` +traces at exactly 1.0. b12x builds without the identity/dynamic compile-spec +fact (see lukealonso/b12x PR "mla: split latent_scale identity/dynamic +compile-cache entries") replay stale identity cubins from persistent compile +caches, silently dropping the restore. Clear mounted b12x compile caches once +when enabling scales on such builds. diff --git a/kv-scales/glm52-nvfp4-nf3-hybrid_mla_outer_scales_v1.json b/kv-scales/glm52-nvfp4-nf3-hybrid_mla_outer_scales_v1.json new file mode 100644 index 000000000000..7ae9d9351c4e --- /dev/null +++ b/kv-scales/glm52-nvfp4-nf3-hybrid_mla_outer_scales_v1.json @@ -0,0 +1,178 @@ +{ + "format": "nvfp4_ds_mla_outer_scale_v1", + "num_layers": 78, + "latent_dim": 512, + "denominator": 2688.0, + "formula": "s_l = max_abs(kv_c_normed) / (6 * 448)", + "model": "madeby561/GLM-5.2-MXFP8-NVFP4-NF3-Hybrid (local K64R16 build; own capture)", + "dataset": { + "name": "Salesforce/wikitext", + "config": "wikitext-2-raw-v1", + "split": "test", + "context_length": 2048, + "windows": 1, + "note": "the canonical festr2-0708 census window" + }, + "created_utc": "2026-07-20T16:40:43.312260+00:00", + "hook": "mla.py kv_a_layernorm output (bind-mount capture patch), per-TP-rank agreement checked", + "max_abs": [ + 0.04833984375, + 0.021728515625, + 0.03662109375, + 0.1005859375, + 0.023681640625, + 0.03515625, + 0.046875, + 0.033203125, + 0.259765625, + 0.1474609375, + 0.7734375, + 1.0, + 1.1171875, + 0.150390625, + 0.76171875, + 0.392578125, + 0.25390625, + 0.455078125, + 0.80859375, + 0.80078125, + 0.96875, + 0.78515625, + 0.58203125, + 0.330078125, + 0.89453125, + 0.73046875, + 1.0390625, + 1.578125, + 1.0078125, + 0.92578125, + 1.25, + 1.5859375, + 1.234375, + 1.9140625, + 1.9453125, + 1.7578125, + 1.7890625, + 2.546875, + 2.296875, + 2.4375, + 1.921875, + 2.171875, + 2.875, + 2.546875, + 2.765625, + 3.234375, + 2.5625, + 3.390625, + 2.40625, + 2.546875, + 3.09375, + 2.90625, + 3.671875, + 2.46875, + 2.609375, + 2.359375, + 3.078125, + 3.171875, + 2.578125, + 2.359375, + 3.890625, + 2.953125, + 4.09375, + 4.09375, + 5.0625, + 5.1875, + 2.984375, + 2.875, + 2.984375, + 3.296875, + 3.828125, + 2.859375, + 3.59375, + 3.734375, + 4.25, + 4.1875, + 4.84375, + 3.75 + ], + "scales": [ + 1.7983572823660715e-05, + 8.083525158110119e-06, + 1.3623918805803572e-05, + 3.742036365327381e-05, + 8.810134161086309e-06, + 1.3078962053571428e-05, + 1.743861607142857e-05, + 1.2352353050595238e-05, + 9.663899739583333e-05, + 5.485897972470238e-05, + 0.00028773716517857144, + 0.0003720238095238095, + 0.00041562034970238094, + 5.5948893229166664e-05, + 0.0002833775111607143, + 0.00014604840959821428, + 9.445917038690477e-05, + 0.00016929989769345238, + 0.00030081612723214287, + 0.0002979096912202381, + 0.0003603980654761905, + 0.00029209681919642856, + 0.00021652948288690475, + 0.0001227969215029762, + 0.00033278692336309525, + 0.00027175176711309525, + 0.0003865559895833333, + 0.0005871000744047619, + 0.0003749302455357143, + 0.0003444126674107143, + 0.0004650297619047619, + 0.0005900065104166666, + 0.0004592168898809524, + 0.0007120768229166666, + 0.0007237025669642857, + 0.0006539481026785714, + 0.0006655738467261905, + 0.0009474981398809524, + 0.0008544921875, + 0.0009068080357142857, + 0.0007149832589285714, + 0.0008079892113095238, + 0.0010695684523809525, + 0.0009474981398809524, + 0.0010288783482142857, + 0.0012032645089285715, + 0.0009533110119047619, + 0.0012613932291666667, + 0.0008951822916666666, + 0.0009474981398809524, + 0.0011509486607142857, + 0.0010811941964285715, + 0.001366024925595238, + 0.0009184337797619048, + 0.0009707496279761905, + 0.0008777436755952381, + 0.0011451357886904762, + 0.0011800130208333333, + 0.0009591238839285714, + 0.0008777436755952381, + 0.0014474051339285715, + 0.0010986328125, + 0.0015229724702380952, + 0.0015229724702380952, + 0.0018833705357142857, + 0.001929873511904762, + 0.001110258556547619, + 0.0010695684523809525, + 0.001110258556547619, + 0.0012265159970238095, + 0.0014241536458333333, + 0.0010637555803571428, + 0.0013369605654761905, + 0.0013892764136904762, + 0.0015811011904761905, + 0.0015578497023809525, + 0.0018019903273809525, + 0.0013950892857142857 + ] +} \ No newline at end of file diff --git a/serve-glm52.sh b/serve-glm52.sh index a34938a21e26..e784311e0468 100755 --- a/serve-glm52.sh +++ b/serve-glm52.sh @@ -107,6 +107,8 @@ MOE_BACKEND="${MOE_BACKEND:-b12x}" MOE_SPEC_BACKEND="${MOE_SPEC_BACKEND:-b12x}" ATTENTION_BACKEND="${ATTENTION_BACKEND:-B12X_MLA_SPARSE}" KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}" +# Calibrated per-layer outer scales for nvfp4_ds_mla KV (empty = disabled). +export VLLM_NVFP4_MLA_SCALES_FILE="${VLLM_NVFP4_MLA_SCALES_FILE:-}" GLM51_PROFILE="${GLM51_PROFILE:-0}" GLM52_CAUSAL_CASCADE="${GLM52_CAUSAL_CASCADE:-0}" GLM52_DSPARK="${GLM52_DSPARK:-0}"