From 5454630b4f1bc22eb82ced668e8a0685970caa05 Mon Sep 17 00:00:00 2001
From: Jiajun Li <guapisolo@gmail.com>
Date: Wed, 14 Oct 2026 13:20:42 -0700
Subject: [PATCH 1/5] ci: stop enforcing per-step grad_norm on the P2P MTP
 tests

The two P2P MTP tests (#3985) landed after small-batch GRPO tests were moved
to shadow grad_norm, so they still enforce it. They log train/grad_norm as
exactly 0 on steps where every GRPO group's rewards are uniform: in their
stored history, GLM-4.7-Flash step 0 is 0.0 in one run and 0.164 in the
other, and Qwen3.5-35B-A3B step 1 is 0.148 and 0.0. The default band cannot
hold both, so the step would fail at random once ten baselines accumulate.
Keep the declarations as shadow checks, like their sibling tests.
---
 .../test_spec_mtptrain_nor3_bf16_sgl_tp2_meg_tp1pp2_p2p.py  | 6 +++++-
 .../test_spec_mtptrain_nor3_bf16_sgl_tp4_meg_tp1cp1_p2p.py  | 6 +++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_nor3_bf16_sgl_tp2_meg_tp1pp2_p2p.py b/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_nor3_bf16_sgl_tp2_meg_tp1pp2_p2p.py
index dbe0613233..5da5c5d4a3 100644
--- a/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_nor3_bf16_sgl_tp2_meg_tp1pp2_p2p.py
+++ b/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_nor3_bf16_sgl_tp2_meg_tp1pp2_p2p.py
@@ -14,7 +14,11 @@
     hardware=["hopper"],
 )
 
-register_ci_gate(metric_key="train/grad_norm")
+register_ci_gate(
+    metric_key="train/grad_norm",
+    enforce=False,
+    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
+)
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff")
 register_ci_gate(metric_key="train/train_rollout_kl")
diff --git a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_mtptrain_nor3_bf16_sgl_tp4_meg_tp1cp1_p2p.py b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_mtptrain_nor3_bf16_sgl_tp4_meg_tp1cp1_p2p.py
index 184217e286..e9edd2796d 100644
--- a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_mtptrain_nor3_bf16_sgl_tp4_meg_tp1cp1_p2p.py
+++ b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_mtptrain_nor3_bf16_sgl_tp4_meg_tp1cp1_p2p.py
@@ -15,7 +15,11 @@
     hardware=["hopper"],
 )
 
-register_ci_gate(metric_key="train/grad_norm")
+register_ci_gate(
+    metric_key="train/grad_norm",
+    enforce=False,
+    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
+)
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff")
 register_ci_gate(metric_key="train/train_rollout_kl")

From bef216cb3b85a543770cf8c340584b35f4864c0d Mon Sep 17 00:00:00 2001
From: Jiajun Li <guapisolo@gmail.com>
Date: Wed, 14 Oct 2026 13:31:08 -0700
Subject: [PATCH 2/5] ci: make the default grad_norm band an order-of-magnitude
 check

train/grad_norm jitters between runs and is exactly 0 on steps where every
GRPO group's rewards are uniform, so the old default band (0.5x up, 0.8x
down, 0.1 floor) failed healthy runs. Twenty tests had opted out with
enforce=False, but a failing shadow check still untrusts the run: 480 of
their 490 untrusted stored runs carry such a failure, so their baselines
for every metric starved. The P2P MTP tests (#3985) still enforced it.

The gate only needs to catch an order-of-magnitude jump. The default now
allows up to 10x the trusted mean (1.0 when that mean is near zero) and any
value down to 0, and every grad_norm declaration is enforced again, which
also reverts the previous commit's per-test opt-out. Replaying all stored
grad_norm history with every run as a baseline, the old band fails 262 of
3347 comparisons with at least ten baselines and the new band fails none.
---
 docs/developer/ci/03-metric-history-gate.md   |  2 +-
 tests/ci/metric_history/register.py           |  4 +++-
 tests/ci/test/test_history_gate.py            | 22 +++++++++++++++++++
 tests/e2e/fsdp/r3/test_glm47_flash_r3.py      |  6 +----
 tests/e2e/fsdp/r3/test_qwen3_30b_a3b_r3.py    |  6 +----
 tests/e2e/fsdp/r3/test_qwen3_5_35b_a3b_r3.py  |  6 +----
 ...p8_deepep_sgl_dpattn1x4_meg_tp2cp1_bshd.py |  6 +----
 ...16_deepep_sgl_dpattn2x2_meg_tp2cp2_bshd.py |  6 +----
 ...ptrain_nor3_bf16_sgl_tp2_meg_tp1pp2_p2p.py |  6 +----
 ..._nor3_fp8_sgl_tp2_meg_tp1cp1_fullyasync.py |  6 +----
 ...train_r3_bf16_sgl_tp2_meg_tp1cp2_disagg.py |  6 +----
 ...pec_mtptrain_r3_bf16_sgl_tp4_meg_tp2pp2.py |  6 +----
 .../test_qwen3_30B_A3B/test_deepep_fp8.py     |  6 +----
 .../test_deepep_fp8_bridge.py                 |  6 +----
 .../test_disagg_broadcast.py                  |  6 +----
 .../test_qwen3_30B_A3B/test_dp_attention.py   |  6 +----
 .../test_qwen3_30B_A3B/test_int4_rollout.py   |  6 +----
 ...est_nospec_nor3_bf16_sgl_tp4_meg_tp2cp2.py |  6 +----
 .../test_qwen3_30B_A3B/test_r3_deepep.py      |  6 +----
 ...r3_bf16_deepep_sgl_dpattn2x2_meg_tp1cp2.py |  6 +----
 ...spec_nor3_bf16_sgl_dpattn2x2_meg_tp2pp2.py |  6 +----
 ...f16_sgl_dpattn1x4_meg_tp1cp1_fullyasync.py |  6 +----
 ...ptrain_nor3_bf16_sgl_tp4_meg_tp1cp1_p2p.py |  6 +----
 ..._r3_fp8_deepep_sgl_dpattn1x4_meg_tp2cp1.py |  6 +----
 .../test_spec_nor3_fp8_sgl_tp4_meg_tp2cp2.py  |  6 +----
 25 files changed, 48 insertions(+), 112 deletions(-)

diff --git a/docs/developer/ci/03-metric-history-gate.md b/docs/developer/ci/03-metric-history-gate.md
index f5bbd8f6e8..1e4c287d6c 100644
--- a/docs/developer/ci/03-metric-history-gate.md
+++ b/docs/developer/ci/03-metric-history-gate.md
@@ -264,7 +264,7 @@ All stored executing suites are kept separate. The replay checks consistency of
 - A test-file edit does not reset the series: `test_file_hash` was dropped from the run-series identity because a tiny edit to a test kept wiping its whole history. A test change that genuinely shifts a metric's expected level surfaces as gate failures instead; the reset lever is manual — `mark_untrusted` the stale runs (the replay's `stale_coordinates` lists them). Editing a declaration's literals keeps the history.
 - Baseline writing is selected by the resolved CI policy rather than inferred from `GITHUB_EVENT_NAME`; nightly and weekly write, while regular and release runs only read history. Enforcement follows each declaration in either mode.
 - One test attempt yields one merged record: a file that launches training more than once concatenates the runs' series, so a step repeats and a `steps="all"` selection is an ERROR. Such a file declares no gate unless each run logs distinct metric keys, as the session TITO tests do.
-- A step value that is exactly zero in some runs and not in others has no stable band. Small-batch GRPO tests log `train/grad_norm` as 0 whenever every group's rewards are uniform, so their `train/grad_norm` declarations set `enforce=False`; the check still decides baseline trust.
+- A step value that is exactly zero in some runs and not in others has no stable band. `train/grad_norm` is the common case: small-batch GRPO tests log it as 0 whenever every group's rewards are uniform, and it jitters between runs otherwise. Its default band is therefore only a sanity check against an order-of-magnitude jump: at most 10× the trusted mean (1.0 when that mean is near zero), any value down to 0.
 - Open: should a brand-new test's first baselines need human confirmation before counting as trusted? (v1: no.)
 
 
diff --git a/tests/ci/metric_history/register.py b/tests/ci/metric_history/register.py
index a975b2909f..d493bf2333 100644
--- a/tests/ci/metric_history/register.py
+++ b/tests/ci/metric_history/register.py
@@ -91,9 +91,11 @@ def register_ci_gate(
 # stay within the capture whitelist (miles.utils.tracking_utils
 # TARGET_METRIC_KEYS; test-enforced, not imported here).
 GATE_DEFAULTS: dict[str, dict] = {
+    # grad_norm jitters and is exactly 0 on steps where every GRPO group's rewards
+    # are uniform, so its band only rejects an order-of-magnitude jump.
     "train/grad_norm": {
         "steps": "all",
-        "constraint": {"rel_up": 0.5, "abs_floor_up": 0.1, "rel_down": 0.8, "abs_floor_down": 0.1},
+        "constraint": {"rel_up": 9.0, "abs_floor_up": 1.0, "rel_down": 1.0},
     },
     "train/ppo_kl": {
         "steps": "all",
diff --git a/tests/ci/test/test_history_gate.py b/tests/ci/test/test_history_gate.py
index 1857d4e471..481b75bdbe 100644
--- a/tests/ci/test/test_history_gate.py
+++ b/tests/ci/test/test_history_gate.py
@@ -608,6 +608,28 @@ def test_reward_corridor_flags_drop_and_suspicious_jump(tmp_path, store):
     )
 
 
+def test_grad_norm_default_flags_only_an_order_of_magnitude_jump(tmp_path, store):
+    # grad_norm jitters and is exactly 0 on steps where every GRPO group's
+    # rewards are uniform, so the default band only rejects a ~10x jump.
+    test_file = _write_test_file(tmp_path, 'register_ci_gate(metric_key="train/grad_norm")')
+    for step, values in ((0, [0.2, 0.2]), (1, [0.0, 0.0])):
+        _seed_baseline(
+            store,
+            test_file,
+            metric_key="train/grad_norm",
+            steps_key=_key("all"),
+            constraint_key="{}",
+            step=step,
+            values=values,
+        )
+    for current, status in ((0.0, GateStatus.PASS), (1.9, GateStatus.PASS), (2.5, GateStatus.FAIL)):
+        record = _write_record(tmp_path, {"train/grad_norm": [[0, current], [1, 0.5]]}, name=f"{current}.jsonl")
+        zero, one = evaluate_gate(test_file, record, store, executing_suite="stage-c-8-gpu-h100").metrics
+        assert (zero.baseline_n, zero.historical_status) == (2, status)
+        # A step whose trusted history is all zeros falls back to the 1.0 floor.
+        assert one.historical_status == GateStatus.PASS
+
+
 # --- multiple specs, no specs ------------------------------------------------
 
 
diff --git a/tests/e2e/fsdp/r3/test_glm47_flash_r3.py b/tests/e2e/fsdp/r3/test_glm47_flash_r3.py
index fabca06f57..bf0c77dd05 100644
--- a/tests/e2e/fsdp/r3/test_glm47_flash_r3.py
+++ b/tests/e2e/fsdp/r3/test_glm47_flash_r3.py
@@ -14,11 +14,7 @@
 register_cuda_ci(est_time=800, suite="stage-c-8-gpu-h200", labels=["fsdp", "replay"], hardware=["hopper"], num_gpus=8)
 register_rocm_ci(est_time=800, suite="nightly-stage-c-8-gpu-mi350", labels=["fsdp", "replay"])
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 # raw_reward averages 64 samples, so it moves in 1/64 steps and healthy runs land up to 0.11 from the mean.
 register_ci_gate(
diff --git a/tests/e2e/fsdp/r3/test_qwen3_30b_a3b_r3.py b/tests/e2e/fsdp/r3/test_qwen3_30b_a3b_r3.py
index 659562f0f4..18c1919e01 100644
--- a/tests/e2e/fsdp/r3/test_qwen3_30b_a3b_r3.py
+++ b/tests/e2e/fsdp/r3/test_qwen3_30b_a3b_r3.py
@@ -6,11 +6,7 @@
 
 register_cuda_ci(est_time=700, suite="stage-c-8-gpu-h200", labels=["fsdp", "replay"], hardware=["hopper"], num_gpus=8)
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="rollout/raw_reward")
 
diff --git a/tests/e2e/fsdp/r3/test_qwen3_5_35b_a3b_r3.py b/tests/e2e/fsdp/r3/test_qwen3_5_35b_a3b_r3.py
index 7ef617db00..f667c1ba26 100644
--- a/tests/e2e/fsdp/r3/test_qwen3_5_35b_a3b_r3.py
+++ b/tests/e2e/fsdp/r3/test_qwen3_5_35b_a3b_r3.py
@@ -6,11 +6,7 @@
 
 register_cuda_ci(est_time=1800, suite="stage-c-8-gpu-h200", labels=["fsdp", "replay"], hardware=["hopper"], num_gpus=8)
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="rollout/raw_reward")
 
diff --git a/tests/e2e/megatron/test_glm47_flash/test_nospec_nor3_fp8_deepep_sgl_dpattn1x4_meg_tp2cp1_bshd.py b/tests/e2e/megatron/test_glm47_flash/test_nospec_nor3_fp8_deepep_sgl_dpattn1x4_meg_tp2cp1_bshd.py
index 4df1556486..c81b45aff8 100644
--- a/tests/e2e/megatron/test_glm47_flash/test_nospec_nor3_fp8_deepep_sgl_dpattn1x4_meg_tp2cp1_bshd.py
+++ b/tests/e2e/megatron/test_glm47_flash/test_nospec_nor3_fp8_deepep_sgl_dpattn1x4_meg_tp2cp1_bshd.py
@@ -15,11 +15,7 @@
 # 4x H200: the 8x H200 runners cannot bring NVSHMEM up over IB, which DeepEP low-latency mode needs.
 register_cuda_ci(est_time=1400, suite="stage-c-4-gpu-h200", labels=["megatron"], hardware=["hopper"], num_gpus=4)
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff")
 register_ci_gate(metric_key="train/train_rollout_kl")
diff --git a/tests/e2e/megatron/test_glm47_flash/test_nospec_r3_bf16_deepep_sgl_dpattn2x2_meg_tp2cp2_bshd.py b/tests/e2e/megatron/test_glm47_flash/test_nospec_r3_bf16_deepep_sgl_dpattn2x2_meg_tp2cp2_bshd.py
index 4697f7dd42..22d3be779c 100644
--- a/tests/e2e/megatron/test_glm47_flash/test_nospec_r3_bf16_deepep_sgl_dpattn2x2_meg_tp2cp2_bshd.py
+++ b/tests/e2e/megatron/test_glm47_flash/test_nospec_r3_bf16_deepep_sgl_dpattn2x2_meg_tp2cp2_bshd.py
@@ -18,11 +18,7 @@
     est_time=1500, suite="stage-c-4-gpu-h200", labels=["megatron", "replay"], hardware=["hopper"], num_gpus=4
 )
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff")
 register_ci_gate(metric_key="train/train_rollout_kl")
diff --git a/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_nor3_bf16_sgl_tp2_meg_tp1pp2_p2p.py b/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_nor3_bf16_sgl_tp2_meg_tp1pp2_p2p.py
index 5da5c5d4a3..dbe0613233 100644
--- a/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_nor3_bf16_sgl_tp2_meg_tp1pp2_p2p.py
+++ b/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_nor3_bf16_sgl_tp2_meg_tp1pp2_p2p.py
@@ -14,11 +14,7 @@
     hardware=["hopper"],
 )
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff")
 register_ci_gate(metric_key="train/train_rollout_kl")
diff --git a/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_nor3_fp8_sgl_tp2_meg_tp1cp1_fullyasync.py b/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_nor3_fp8_sgl_tp2_meg_tp1cp1_fullyasync.py
index 7e17d11048..2430957a42 100644
--- a/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_nor3_fp8_sgl_tp2_meg_tp1cp1_fullyasync.py
+++ b/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_nor3_fp8_sgl_tp2_meg_tp1cp1_fullyasync.py
@@ -19,11 +19,7 @@
     num_gpus=8,
 )
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff")
 register_ci_gate(metric_key="train/train_rollout_kl")
diff --git a/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_r3_bf16_sgl_tp2_meg_tp1cp2_disagg.py b/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_r3_bf16_sgl_tp2_meg_tp1cp2_disagg.py
index 528e5703ab..97eef9e5f2 100644
--- a/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_r3_bf16_sgl_tp2_meg_tp1cp2_disagg.py
+++ b/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_r3_bf16_sgl_tp2_meg_tp1cp2_disagg.py
@@ -12,11 +12,7 @@
     num_gpus=4,
 )
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff")
 register_ci_gate(metric_key="train/train_rollout_kl")
diff --git a/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_r3_bf16_sgl_tp4_meg_tp2pp2.py b/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_r3_bf16_sgl_tp4_meg_tp2pp2.py
index 830f761dfb..cbde0ea5a3 100644
--- a/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_r3_bf16_sgl_tp4_meg_tp2pp2.py
+++ b/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_r3_bf16_sgl_tp4_meg_tp2pp2.py
@@ -9,11 +9,7 @@
 )
 register_rocm_ci(est_time=1500, suite="nightly-stage-c-4-gpu-mi350", labels=["megatron"])
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff")
 register_ci_gate(metric_key="train/train_rollout_kl")
diff --git a/tests/e2e/megatron/test_qwen3_30B_A3B/test_deepep_fp8.py b/tests/e2e/megatron/test_qwen3_30B_A3B/test_deepep_fp8.py
index 907051d03a..e759ebe95c 100644
--- a/tests/e2e/megatron/test_qwen3_30B_A3B/test_deepep_fp8.py
+++ b/tests/e2e/megatron/test_qwen3_30B_A3B/test_deepep_fp8.py
@@ -8,11 +8,7 @@
     est_time=1500, suite="stage-c-4-gpu-h200", labels=["megatron"], hardware=["hopper", "blackwell"], num_gpus=4
 )
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff")
 register_ci_gate(metric_key="train/train_rollout_kl")
diff --git a/tests/e2e/megatron/test_qwen3_30B_A3B/test_deepep_fp8_bridge.py b/tests/e2e/megatron/test_qwen3_30B_A3B/test_deepep_fp8_bridge.py
index c0c8b6931b..3ff7e43c41 100644
--- a/tests/e2e/megatron/test_qwen3_30B_A3B/test_deepep_fp8_bridge.py
+++ b/tests/e2e/megatron/test_qwen3_30B_A3B/test_deepep_fp8_bridge.py
@@ -9,11 +9,7 @@
     est_time=1300, suite="stage-c-4-gpu-h200", labels=["megatron"], hardware=["hopper", "blackwell"], num_gpus=4
 )
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff")
 register_ci_gate(metric_key="train/train_rollout_kl")
diff --git a/tests/e2e/megatron/test_qwen3_30B_A3B/test_disagg_broadcast.py b/tests/e2e/megatron/test_qwen3_30B_A3B/test_disagg_broadcast.py
index 41b5220b35..7d2fc980fe 100644
--- a/tests/e2e/megatron/test_qwen3_30B_A3B/test_disagg_broadcast.py
+++ b/tests/e2e/megatron/test_qwen3_30B_A3B/test_disagg_broadcast.py
@@ -13,11 +13,7 @@
 )
 register_rocm_ci(est_time=900, suite="nightly-stage-c-4-gpu-mi350", labels=["megatron", "weight-update"])
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff")
 register_ci_gate(metric_key="train/train_rollout_kl")
diff --git a/tests/e2e/megatron/test_qwen3_30B_A3B/test_dp_attention.py b/tests/e2e/megatron/test_qwen3_30B_A3B/test_dp_attention.py
index d57dd487a8..71133c99ae 100644
--- a/tests/e2e/megatron/test_qwen3_30B_A3B/test_dp_attention.py
+++ b/tests/e2e/megatron/test_qwen3_30B_A3B/test_dp_attention.py
@@ -21,11 +21,7 @@
 )
 register_rocm_ci(est_time=800, suite="nightly-stage-c-4-gpu-mi350", labels=["megatron", "short"])
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff")
 register_ci_gate(metric_key="train/train_rollout_kl")
diff --git a/tests/e2e/megatron/test_qwen3_30B_A3B/test_int4_rollout.py b/tests/e2e/megatron/test_qwen3_30B_A3B/test_int4_rollout.py
index 2a3da0f51e..2c6fdf1d0e 100644
--- a/tests/e2e/megatron/test_qwen3_30B_A3B/test_int4_rollout.py
+++ b/tests/e2e/megatron/test_qwen3_30B_A3B/test_int4_rollout.py
@@ -8,11 +8,7 @@
     est_time=1700, suite="stage-c-4-gpu-h200", labels=["megatron"], hardware=["hopper", "blackwell"], num_gpus=4
 )
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff")
 register_ci_gate(metric_key="train/train_rollout_kl")
diff --git a/tests/e2e/megatron/test_qwen3_30B_A3B/test_nospec_nor3_bf16_sgl_tp4_meg_tp2cp2.py b/tests/e2e/megatron/test_qwen3_30B_A3B/test_nospec_nor3_bf16_sgl_tp4_meg_tp2cp2.py
index 2413c512dc..54bac9591a 100644
--- a/tests/e2e/megatron/test_qwen3_30B_A3B/test_nospec_nor3_bf16_sgl_tp4_meg_tp2cp2.py
+++ b/tests/e2e/megatron/test_qwen3_30B_A3B/test_nospec_nor3_bf16_sgl_tp4_meg_tp2cp2.py
@@ -14,11 +14,7 @@
 )
 register_rocm_ci(est_time=1000, suite="nightly-stage-c-4-gpu-mi350", labels=["megatron"])
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff")
 register_ci_gate(metric_key="train/train_rollout_kl")
diff --git a/tests/e2e/megatron/test_qwen3_30B_A3B/test_r3_deepep.py b/tests/e2e/megatron/test_qwen3_30B_A3B/test_r3_deepep.py
index 339bf9b84e..bb36f9e02e 100644
--- a/tests/e2e/megatron/test_qwen3_30B_A3B/test_r3_deepep.py
+++ b/tests/e2e/megatron/test_qwen3_30B_A3B/test_r3_deepep.py
@@ -14,11 +14,7 @@
     num_gpus=4,
 )
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff")
 register_ci_gate(metric_key="train/train_rollout_kl")
diff --git a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_deepep_sgl_dpattn2x2_meg_tp1cp2.py b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_deepep_sgl_dpattn2x2_meg_tp1cp2.py
index eb17017603..ed072c42a9 100644
--- a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_deepep_sgl_dpattn2x2_meg_tp1cp2.py
+++ b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_deepep_sgl_dpattn2x2_meg_tp1cp2.py
@@ -15,11 +15,7 @@
     est_time=1500, suite="stage-c-4-gpu-h200", labels=["megatron", "qwen35"], hardware=["hopper"], num_gpus=4
 )
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff")
 register_ci_gate(metric_key="train/train_rollout_kl")
diff --git a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_sgl_dpattn2x2_meg_tp2pp2.py b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_sgl_dpattn2x2_meg_tp2pp2.py
index 2867d8589c..34f3835519 100644
--- a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_sgl_dpattn2x2_meg_tp2pp2.py
+++ b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_sgl_dpattn2x2_meg_tp2pp2.py
@@ -18,11 +18,7 @@
 )
 register_rocm_ci(est_time=1300, suite="nightly-stage-c-4-gpu-mi350", labels=["megatron", "qwen35"])
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff")
 register_ci_gate(metric_key="train/train_rollout_kl")
diff --git a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_r3_bf16_sgl_dpattn1x4_meg_tp1cp1_fullyasync.py b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_r3_bf16_sgl_dpattn1x4_meg_tp1cp1_fullyasync.py
index 498a1d7059..127b1675ca 100644
--- a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_r3_bf16_sgl_dpattn1x4_meg_tp1cp1_fullyasync.py
+++ b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_r3_bf16_sgl_dpattn1x4_meg_tp1cp1_fullyasync.py
@@ -27,11 +27,7 @@
     labels=["megatron", "qwen35", "weight-update", "fully-async", "replay"],
 )
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff")
 register_ci_gate(metric_key="train/train_rollout_kl")
diff --git a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_mtptrain_nor3_bf16_sgl_tp4_meg_tp1cp1_p2p.py b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_mtptrain_nor3_bf16_sgl_tp4_meg_tp1cp1_p2p.py
index e9edd2796d..184217e286 100644
--- a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_mtptrain_nor3_bf16_sgl_tp4_meg_tp1cp1_p2p.py
+++ b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_mtptrain_nor3_bf16_sgl_tp4_meg_tp1cp1_p2p.py
@@ -15,11 +15,7 @@
     hardware=["hopper"],
 )
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff")
 register_ci_gate(metric_key="train/train_rollout_kl")
diff --git a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_mtptrain_r3_fp8_deepep_sgl_dpattn1x4_meg_tp2cp1.py b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_mtptrain_r3_fp8_deepep_sgl_dpattn1x4_meg_tp2cp1.py
index 71ca9ea043..28e0d918fd 100644
--- a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_mtptrain_r3_fp8_deepep_sgl_dpattn1x4_meg_tp2cp1.py
+++ b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_mtptrain_r3_fp8_deepep_sgl_dpattn1x4_meg_tp2cp1.py
@@ -18,11 +18,7 @@
     est_time=2200, suite="stage-c-4-gpu-h200", labels=["megatron", "qwen35", "replay"], hardware=["hopper"], num_gpus=4
 )
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff")
 register_ci_gate(metric_key="train/train_rollout_kl")
diff --git a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_nor3_fp8_sgl_tp4_meg_tp2cp2.py b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_nor3_fp8_sgl_tp4_meg_tp2cp2.py
index 25156a3195..45c2c71d74 100644
--- a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_nor3_fp8_sgl_tp4_meg_tp2cp2.py
+++ b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_nor3_fp8_sgl_tp4_meg_tp2cp2.py
@@ -27,11 +27,7 @@
 )
 register_rocm_ci(est_time=1600, suite="nightly-stage-c-4-gpu-mi350", labels=["megatron", "qwen35"])
 
-register_ci_gate(
-    metric_key="train/grad_norm",
-    enforce=False,
-    allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.",
-)
+register_ci_gate(metric_key="train/grad_norm")
 register_ci_gate(metric_key="train/ppo_kl")
 register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff")
 register_ci_gate(metric_key="train/train_rollout_kl")

From 571e2382e65782c4717e68f7973cc52fc722230a Mon Sep 17 00:00:00 2001
From: Jiajun Li <guapisolo@gmail.com>
Date: Wed, 14 Oct 2026 13:47:14 -0700
Subject: [PATCH 3/5] ci: state both limits of the grad_norm sanity band

The default grad_norm band passes a value up to 10x the trusted mean or the
mean plus 1.0, whichever is higher, so anything up to 1.0 passes. The
comment and docs said "1.0 when the mean is near zero"; spell out the rule
and why it is wide: exact-zero steps also drag the averaged baseline down.
The test now covers a small mean, a mean above 1.0, and an all-zero history,
and fails under a 2x band as well as under the old one.
---
 docs/developer/ci/03-metric-history-gate.md |  2 +-
 tests/ci/metric_history/register.py         |  3 ++-
 tests/ci/test/test_history_gate.py          | 24 ++++++++++++---------
 3 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/docs/developer/ci/03-metric-history-gate.md b/docs/developer/ci/03-metric-history-gate.md
index 1e4c287d6c..43362f0e5a 100644
--- a/docs/developer/ci/03-metric-history-gate.md
+++ b/docs/developer/ci/03-metric-history-gate.md
@@ -264,7 +264,7 @@ All stored executing suites are kept separate. The replay checks consistency of
 - A test-file edit does not reset the series: `test_file_hash` was dropped from the run-series identity because a tiny edit to a test kept wiping its whole history. A test change that genuinely shifts a metric's expected level surfaces as gate failures instead; the reset lever is manual — `mark_untrusted` the stale runs (the replay's `stale_coordinates` lists them). Editing a declaration's literals keeps the history.
 - Baseline writing is selected by the resolved CI policy rather than inferred from `GITHUB_EVENT_NAME`; nightly and weekly write, while regular and release runs only read history. Enforcement follows each declaration in either mode.
 - One test attempt yields one merged record: a file that launches training more than once concatenates the runs' series, so a step repeats and a `steps="all"` selection is an ERROR. Such a file declares no gate unless each run logs distinct metric keys, as the session TITO tests do.
-- A step value that is exactly zero in some runs and not in others has no stable band. `train/grad_norm` is the common case: small-batch GRPO tests log it as 0 whenever every group's rewards are uniform, and it jitters between runs otherwise. Its default band is therefore only a sanity check against an order-of-magnitude jump: at most 10× the trusted mean (1.0 when that mean is near zero), any value down to 0.
+- A step value that is exactly zero in some runs and not in others has no stable band. `train/grad_norm` is the common case: small-batch GRPO tests log it as 0 whenever every group's rewards are uniform, which also drags the averaged baseline down, and it jitters between runs otherwise. Its default band is therefore only a sanity check: a value passes up to 10× the trusted mean or the mean plus 1.0, whichever is higher (so anything up to 1.0 passes), and down to 0.
 - Open: should a brand-new test's first baselines need human confirmation before counting as trusted? (v1: no.)
 
 
diff --git a/tests/ci/metric_history/register.py b/tests/ci/metric_history/register.py
index d493bf2333..03215acb2b 100644
--- a/tests/ci/metric_history/register.py
+++ b/tests/ci/metric_history/register.py
@@ -92,7 +92,8 @@ def register_ci_gate(
 # TARGET_METRIC_KEYS; test-enforced, not imported here).
 GATE_DEFAULTS: dict[str, dict] = {
     # grad_norm jitters and is exactly 0 on steps where every GRPO group's rewards
-    # are uniform, so its band only rejects an order-of-magnitude jump.
+    # are uniform, which also drags the averaged baseline down. The band is only a
+    # sanity check: up to 10x the mean or the mean plus 1.0, whichever is higher.
     "train/grad_norm": {
         "steps": "all",
         "constraint": {"rel_up": 9.0, "abs_floor_up": 1.0, "rel_down": 1.0},
diff --git a/tests/ci/test/test_history_gate.py b/tests/ci/test/test_history_gate.py
index 481b75bdbe..f2c67b0829 100644
--- a/tests/ci/test/test_history_gate.py
+++ b/tests/ci/test/test_history_gate.py
@@ -608,11 +608,11 @@ def test_reward_corridor_flags_drop_and_suspicious_jump(tmp_path, store):
     )
 
 
-def test_grad_norm_default_flags_only_an_order_of_magnitude_jump(tmp_path, store):
-    # grad_norm jitters and is exactly 0 on steps where every GRPO group's
-    # rewards are uniform, so the default band only rejects a ~10x jump.
+def test_grad_norm_default_is_a_sanity_band(tmp_path, store):
+    # grad_norm jitters and is exactly 0 on steps where every GRPO group's rewards
+    # are uniform, so a value passes up to max(10x the mean, mean + 1.0), down to 0.
     test_file = _write_test_file(tmp_path, 'register_ci_gate(metric_key="train/grad_norm")')
-    for step, values in ((0, [0.2, 0.2]), (1, [0.0, 0.0])):
+    for step, values in ((0, [0.2, 0.2]), (1, [2.0, 2.0]), (2, [0.0, 0.0])):
         _seed_baseline(
             store,
             test_file,
@@ -622,12 +622,16 @@ def test_grad_norm_default_flags_only_an_order_of_magnitude_jump(tmp_path, store
             step=step,
             values=values,
         )
-    for current, status in ((0.0, GateStatus.PASS), (1.9, GateStatus.PASS), (2.5, GateStatus.FAIL)):
-        record = _write_record(tmp_path, {"train/grad_norm": [[0, current], [1, 0.5]]}, name=f"{current}.jsonl")
-        zero, one = evaluate_gate(test_file, record, store, executing_suite="stage-c-8-gpu-h100").metrics
-        assert (zero.baseline_n, zero.historical_status) == (2, status)
-        # A step whose trusted history is all zeros falls back to the 1.0 floor.
-        assert one.historical_status == GateStatus.PASS
+    cases = [
+        ((0.0, 0.0, 0.0), GateStatus.PASS),
+        ((1.9, 19.0, 0.9), GateStatus.PASS),
+        ((2.5, 21.0, 1.1), GateStatus.FAIL),
+    ]
+    for currents, status in cases:
+        series = [[step, value] for step, value in enumerate(currents)]
+        record = _write_record(tmp_path, {"train/grad_norm": series}, name=f"{currents[0]}.jsonl")
+        metrics = evaluate_gate(test_file, record, store, executing_suite="stage-c-8-gpu-h100").metrics
+        assert [(m.baseline_n, m.historical_status) for m in metrics] == [(2, status)] * 3
 
 
 # --- multiple specs, no specs ------------------------------------------------

From ec92413ea57e806bcf64a75941d994767a6dffb3 Mon Sep 17 00:00:00 2001
From: Jiajun Li <guapisolo@gmail.com>
Date: Wed, 14 Oct 2026 14:00:58 -0700
Subject: [PATCH 4/5] ci: cap the grad_norm sanity band at 4x the mean

A 4x cap still passes every stored grad_norm value. Replaying the nightly
history in time order with trust re-derived per run, grad_norm fails 0
times and its largest value reaches 0.80 of the limit (0.977 against a
0.22 mean, under the mean + 1.0 floor); the largest jump above 1.0, a
kimi_k3 level shift from 1.55 to 2.84, uses 46% of it. A 4x jump in a
test whose grad_norm sits above 1.0 now fails. The test boundaries move
with the cap and fail under the old band, a 2x band and a 10x band.
---
 docs/developer/ci/03-metric-history-gate.md | 2 +-
 tests/ci/metric_history/register.py         | 4 ++--
 tests/ci/test/test_history_gate.py          | 6 +++---
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/developer/ci/03-metric-history-gate.md b/docs/developer/ci/03-metric-history-gate.md
index 43362f0e5a..dbda8c7d65 100644
--- a/docs/developer/ci/03-metric-history-gate.md
+++ b/docs/developer/ci/03-metric-history-gate.md
@@ -264,7 +264,7 @@ All stored executing suites are kept separate. The replay checks consistency of
 - A test-file edit does not reset the series: `test_file_hash` was dropped from the run-series identity because a tiny edit to a test kept wiping its whole history. A test change that genuinely shifts a metric's expected level surfaces as gate failures instead; the reset lever is manual — `mark_untrusted` the stale runs (the replay's `stale_coordinates` lists them). Editing a declaration's literals keeps the history.
 - Baseline writing is selected by the resolved CI policy rather than inferred from `GITHUB_EVENT_NAME`; nightly and weekly write, while regular and release runs only read history. Enforcement follows each declaration in either mode.
 - One test attempt yields one merged record: a file that launches training more than once concatenates the runs' series, so a step repeats and a `steps="all"` selection is an ERROR. Such a file declares no gate unless each run logs distinct metric keys, as the session TITO tests do.
-- A step value that is exactly zero in some runs and not in others has no stable band. `train/grad_norm` is the common case: small-batch GRPO tests log it as 0 whenever every group's rewards are uniform, which also drags the averaged baseline down, and it jitters between runs otherwise. Its default band is therefore only a sanity check: a value passes up to 10× the trusted mean or the mean plus 1.0, whichever is higher (so anything up to 1.0 passes), and down to 0.
+- A step value that is exactly zero in some runs and not in others has no stable band. `train/grad_norm` is the common case: small-batch GRPO tests log it as 0 whenever every group's rewards are uniform, which also drags the averaged baseline down, and it jitters between runs otherwise. Its default band is therefore only a sanity check: a value passes up to 4× the trusted mean or the mean plus 1.0, whichever is higher (so anything up to 1.0 passes), and down to 0.
 - Open: should a brand-new test's first baselines need human confirmation before counting as trusted? (v1: no.)
 
 
diff --git a/tests/ci/metric_history/register.py b/tests/ci/metric_history/register.py
index 03215acb2b..951f755bef 100644
--- a/tests/ci/metric_history/register.py
+++ b/tests/ci/metric_history/register.py
@@ -93,10 +93,10 @@ def register_ci_gate(
 GATE_DEFAULTS: dict[str, dict] = {
     # grad_norm jitters and is exactly 0 on steps where every GRPO group's rewards
     # are uniform, which also drags the averaged baseline down. The band is only a
-    # sanity check: up to 10x the mean or the mean plus 1.0, whichever is higher.
+    # sanity check: up to 4x the mean or the mean plus 1.0, whichever is higher.
     "train/grad_norm": {
         "steps": "all",
-        "constraint": {"rel_up": 9.0, "abs_floor_up": 1.0, "rel_down": 1.0},
+        "constraint": {"rel_up": 3.0, "abs_floor_up": 1.0, "rel_down": 1.0},
     },
     "train/ppo_kl": {
         "steps": "all",
diff --git a/tests/ci/test/test_history_gate.py b/tests/ci/test/test_history_gate.py
index f2c67b0829..125388170b 100644
--- a/tests/ci/test/test_history_gate.py
+++ b/tests/ci/test/test_history_gate.py
@@ -610,7 +610,7 @@ def test_reward_corridor_flags_drop_and_suspicious_jump(tmp_path, store):
 
 def test_grad_norm_default_is_a_sanity_band(tmp_path, store):
     # grad_norm jitters and is exactly 0 on steps where every GRPO group's rewards
-    # are uniform, so a value passes up to max(10x the mean, mean + 1.0), down to 0.
+    # are uniform, so a value passes up to max(4x the mean, mean + 1.0), down to 0.
     test_file = _write_test_file(tmp_path, 'register_ci_gate(metric_key="train/grad_norm")')
     for step, values in ((0, [0.2, 0.2]), (1, [2.0, 2.0]), (2, [0.0, 0.0])):
         _seed_baseline(
@@ -624,8 +624,8 @@ def test_grad_norm_default_is_a_sanity_band(tmp_path, store):
         )
     cases = [
         ((0.0, 0.0, 0.0), GateStatus.PASS),
-        ((1.9, 19.0, 0.9), GateStatus.PASS),
-        ((2.5, 21.0, 1.1), GateStatus.FAIL),
+        ((1.1, 7.9, 0.9), GateStatus.PASS),
+        ((1.3, 8.1, 1.1), GateStatus.FAIL),
     ]
     for currents, status in cases:
         series = [[step, value] for step, value in enumerate(currents)]

From 8be6d1576c1e54cbeacd7044b7dd40338cc43ee6 Mon Sep 17 00:00:00 2001
From: Jiajun Li <guapisolo@gmail.com>
Date: Wed, 14 Oct 2026 14:09:08 -0700
Subject: [PATCH 5/5] docs(ci): stop suggesting enforce=False for zero-flip
 coordinates

A failing check keeps its run out of trusted history even with
enforce=False, which is how the twenty grad_norm overrides starved their
tests' baselines. Say so where enforce=False is described, and point
zero-flip coordinates at a wider band instead, as the grad_norm default
now does.
---
 docs/developer/ci/03-metric-history-gate.md | 4 ++--
 tests/ci/metric_history/replay.py           | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/developer/ci/03-metric-history-gate.md b/docs/developer/ci/03-metric-history-gate.md
index dbda8c7d65..860a4a3f10 100644
--- a/docs/developer/ci/03-metric-history-gate.md
+++ b/docs/developer/ci/03-metric-history-gate.md
@@ -219,7 +219,7 @@ Use $neon-access to show the 10 most recent metric-history runs for tests/e2e/me
 
 **Goal:** automatically fail CI once a coordinate has enough trusted history.
 
-Every `register_ci_gate(...)` declaration defaults to `enforce=True`; tests and PRs need no opt-in marker or allowlist. Numerical regressions fail CI once the coordinate has **10 trusted baseline values**. The existing explicit `enforce=False` override is an escape hatch, not the rollout mechanism. Policy metadata never changes the baseline key or comparison band.
+Every `register_ci_gate(...)` declaration defaults to `enforce=True`; tests and PRs need no opt-in marker or allowlist. Numerical regressions fail CI once the coordinate has **10 trusted baseline values**. The existing explicit `enforce=False` override is an escape hatch, not the rollout mechanism: it stops the CI failure, but a failing check still keeps the run out of trusted history. Policy metadata never changes the baseline key or comparison band.
 
 - Below 10 trusted values, the comparison still controls baseline trust but does not fail CI for numerical drift; the summary explicitly reports insufficient history. Each step and executing suite is counted independently.
 - Missing, empty, invalid or non-finite metrics fail tests with enforcement enabled. With a configured store, record-merge, gate-evaluation and database errors also fail instead of silently bypassing the gate.
@@ -251,7 +251,7 @@ The replay checks each stored value, including untrusted candidates, with the pr
 - **Baseline:** up to 20 strictly earlier trusted values of the identical test/backend/executing-suite/metric/step coordinate, whatever declaration literals judged them; values stored at points the current `steps` selection does not choose are not part of its history. Current and future rows never enter a candidate's baseline; rows with the same timestamp cannot establish an ordering and are excluded from each other's history. A tie split by the 20-value window boundary is rejected because the production query does not specify a tie-breaker.
 - **Coverage:** zero-history values are cold starts, not passing checks. Comparisons with one to nine predecessors still check historical trust consistency, but are marked insufficient for enforcement. Each run is also checked for missing declared metrics and explicitly required steps, starting from the coordinate's first stored value: an older run predates the declaration and is not missing it, and a coordinate with no stored value surfaces as a declaration without enforcement checks. For `steps="all"`, the stored snapshot cannot reveal absent original steps, so only declaration presence and observed steps can be checked.
 - **Report:** passes, failures, enforcement-eligible comparisons, insufficient history, incomplete stored values, and declarations without any enforcement-eligible comparison are reported separately; failures or incomplete coverage return a nonzero exit code. A malformed, truncated, or differently scoped export is rejected.
-- **History health:** `stale_coordinates` lists coordinates whose latest three comparisons all fail on the same side of the baseline, the pattern of a trusted history that no longer matches the current level; `reset_run_ids` are that coordinate's trusted runs to `mark_untrusted` (a whole-run flag, so the run's other coordinates also lose these values and may cold-start or fall back to older trusted runs). `zero_flip_coordinates` lists failing coordinates whose values mix exact zeros with nonzero values; no band may fit them, and `enforce=False` is the usual fix. A coordinate in both lists is ambiguous: read its values before choosing a reset or `enforce=False`.
+- **History health:** `stale_coordinates` lists coordinates whose latest three comparisons all fail on the same side of the baseline, the pattern of a trusted history that no longer matches the current level; `reset_run_ids` are that coordinate's trusted runs to `mark_untrusted` (a whole-run flag, so the run's other coordinates also lose these values and may cold-start or fall back to older trusted runs). `zero_flip_coordinates` lists failing coordinates whose values mix exact zeros with nonzero values; no tight band fits them, and `enforce=False` would still leave their runs untrusted, so widen the band as the `train/grad_norm` default does. A coordinate in both lists is ambiguous: read its values before choosing a reset or a wider band.
 
 Only earlier rows with the snapshot's `trusted=True` flag enter the baseline; candidate rows are never filtered by trust. The report separates trusted-candidate inconsistencies from untrusted-candidate regressions. The replay uses the snapshot's current `trusted` flags; it does not reconstruct later trust revocations or historical code versions, infer missing original metrics from stored values, rewrite trust, or change which rows enter subsequent history.
 
diff --git a/tests/ci/metric_history/replay.py b/tests/ci/metric_history/replay.py
index e880f8d653..69e7048e71 100644
--- a/tests/ci/metric_history/replay.py
+++ b/tests/ci/metric_history/replay.py
@@ -246,8 +246,8 @@ def replay(rows: list[dict], specs: dict[tuple[str, str], CiGateSpec]) -> dict:
                 )
             )
         if stats["failed"] and 0 in values and any(values):
-            # Exact zeros beside nonzero values may leave no band to tune, so
-            # enforce=False may fit better than a history reset.
+            # Exact zeros beside nonzero values may leave no tight band, so a
+            # wider band may fit better than a history reset.
             zero_flips.append(
                 dict(**coordinate, zero_values=values.count(0), nonzero_values=len(values) - values.count(0))
             )