diff --git a/docs/developer/ci/03-metric-history-gate.md b/docs/developer/ci/03-metric-history-gate.md index f5bbd8f6e8..860a4a3f10 100644 --- a/docs/developer/ci/03-metric-history-gate.md +++ b/docs/developer/ci/03-metric-history-gate.md @@ -219,7 +219,7 @@ Use $neon-access to show the 10 most recent metric-history runs for tests/e2e/me **Goal:** automatically fail CI once a coordinate has enough trusted history. -Every `register_ci_gate(...)` declaration defaults to `enforce=True`; tests and PRs need no opt-in marker or allowlist. Numerical regressions fail CI once the coordinate has **10 trusted baseline values**. The existing explicit `enforce=False` override is an escape hatch, not the rollout mechanism. Policy metadata never changes the baseline key or comparison band. +Every `register_ci_gate(...)` declaration defaults to `enforce=True`; tests and PRs need no opt-in marker or allowlist. Numerical regressions fail CI once the coordinate has **10 trusted baseline values**. The existing explicit `enforce=False` override is an escape hatch, not the rollout mechanism: it stops the CI failure, but a failing check still keeps the run out of trusted history. Policy metadata never changes the baseline key or comparison band. - Below 10 trusted values, the comparison still controls baseline trust but does not fail CI for numerical drift; the summary explicitly reports insufficient history. Each step and executing suite is counted independently. - Missing, empty, invalid or non-finite metrics fail tests with enforcement enabled. With a configured store, record-merge, gate-evaluation and database errors also fail instead of silently bypassing the gate. @@ -251,7 +251,7 @@ The replay checks each stored value, including untrusted candidates, with the pr - **Baseline:** up to 20 strictly earlier trusted values of the identical test/backend/executing-suite/metric/step coordinate, whatever declaration literals judged them; values stored at points the current `steps` selection does not choose are not part of its history. Current and future rows never enter a candidate's baseline; rows with the same timestamp cannot establish an ordering and are excluded from each other's history. A tie split by the 20-value window boundary is rejected because the production query does not specify a tie-breaker. - **Coverage:** zero-history values are cold starts, not passing checks. Comparisons with one to nine predecessors still check historical trust consistency, but are marked insufficient for enforcement. Each run is also checked for missing declared metrics and explicitly required steps, starting from the coordinate's first stored value: an older run predates the declaration and is not missing it, and a coordinate with no stored value surfaces as a declaration without enforcement checks. For `steps="all"`, the stored snapshot cannot reveal absent original steps, so only declaration presence and observed steps can be checked. - **Report:** passes, failures, enforcement-eligible comparisons, insufficient history, incomplete stored values, and declarations without any enforcement-eligible comparison are reported separately; failures or incomplete coverage return a nonzero exit code. A malformed, truncated, or differently scoped export is rejected. -- **History health:** `stale_coordinates` lists coordinates whose latest three comparisons all fail on the same side of the baseline, the pattern of a trusted history that no longer matches the current level; `reset_run_ids` are that coordinate's trusted runs to `mark_untrusted` (a whole-run flag, so the run's other coordinates also lose these values and may cold-start or fall back to older trusted runs). `zero_flip_coordinates` lists failing coordinates whose values mix exact zeros with nonzero values; no band may fit them, and `enforce=False` is the usual fix. A coordinate in both lists is ambiguous: read its values before choosing a reset or `enforce=False`. +- **History health:** `stale_coordinates` lists coordinates whose latest three comparisons all fail on the same side of the baseline, the pattern of a trusted history that no longer matches the current level; `reset_run_ids` are that coordinate's trusted runs to `mark_untrusted` (a whole-run flag, so the run's other coordinates also lose these values and may cold-start or fall back to older trusted runs). `zero_flip_coordinates` lists failing coordinates whose values mix exact zeros with nonzero values; no tight band fits them, and `enforce=False` would still leave their runs untrusted, so widen the band as the `train/grad_norm` default does. A coordinate in both lists is ambiguous: read its values before choosing a reset or a wider band. Only earlier rows with the snapshot's `trusted=True` flag enter the baseline; candidate rows are never filtered by trust. The report separates trusted-candidate inconsistencies from untrusted-candidate regressions. The replay uses the snapshot's current `trusted` flags; it does not reconstruct later trust revocations or historical code versions, infer missing original metrics from stored values, rewrite trust, or change which rows enter subsequent history. @@ -264,7 +264,7 @@ All stored executing suites are kept separate. The replay checks consistency of - A test-file edit does not reset the series: `test_file_hash` was dropped from the run-series identity because a tiny edit to a test kept wiping its whole history. A test change that genuinely shifts a metric's expected level surfaces as gate failures instead; the reset lever is manual — `mark_untrusted` the stale runs (the replay's `stale_coordinates` lists them). Editing a declaration's literals keeps the history. - Baseline writing is selected by the resolved CI policy rather than inferred from `GITHUB_EVENT_NAME`; nightly and weekly write, while regular and release runs only read history. Enforcement follows each declaration in either mode. - One test attempt yields one merged record: a file that launches training more than once concatenates the runs' series, so a step repeats and a `steps="all"` selection is an ERROR. Such a file declares no gate unless each run logs distinct metric keys, as the session TITO tests do. -- A step value that is exactly zero in some runs and not in others has no stable band. Small-batch GRPO tests log `train/grad_norm` as 0 whenever every group's rewards are uniform, so their `train/grad_norm` declarations set `enforce=False`; the check still decides baseline trust. +- A step value that is exactly zero in some runs and not in others has no stable band. `train/grad_norm` is the common case: small-batch GRPO tests log it as 0 whenever every group's rewards are uniform, which also drags the averaged baseline down, and it jitters between runs otherwise. Its default band is therefore only a sanity check: a value passes up to 4× the trusted mean or the mean plus 1.0, whichever is higher (so anything up to 1.0 passes), and down to 0. - Open: should a brand-new test's first baselines need human confirmation before counting as trusted? (v1: no.) diff --git a/tests/ci/metric_history/register.py b/tests/ci/metric_history/register.py index a975b2909f..951f755bef 100644 --- a/tests/ci/metric_history/register.py +++ b/tests/ci/metric_history/register.py @@ -91,9 +91,12 @@ def register_ci_gate( # stay within the capture whitelist (miles.utils.tracking_utils # TARGET_METRIC_KEYS; test-enforced, not imported here). GATE_DEFAULTS: dict[str, dict] = { + # grad_norm jitters and is exactly 0 on steps where every GRPO group's rewards + # are uniform, which also drags the averaged baseline down. The band is only a + # sanity check: up to 4x the mean or the mean plus 1.0, whichever is higher. "train/grad_norm": { "steps": "all", - "constraint": {"rel_up": 0.5, "abs_floor_up": 0.1, "rel_down": 0.8, "abs_floor_down": 0.1}, + "constraint": {"rel_up": 3.0, "abs_floor_up": 1.0, "rel_down": 1.0}, }, "train/ppo_kl": { "steps": "all", diff --git a/tests/ci/metric_history/replay.py b/tests/ci/metric_history/replay.py index e880f8d653..69e7048e71 100644 --- a/tests/ci/metric_history/replay.py +++ b/tests/ci/metric_history/replay.py @@ -246,8 +246,8 @@ def replay(rows: list[dict], specs: dict[tuple[str, str], CiGateSpec]) -> dict: ) ) if stats["failed"] and 0 in values and any(values): - # Exact zeros beside nonzero values may leave no band to tune, so - # enforce=False may fit better than a history reset. + # Exact zeros beside nonzero values may leave no tight band, so a + # wider band may fit better than a history reset. zero_flips.append( dict(**coordinate, zero_values=values.count(0), nonzero_values=len(values) - values.count(0)) ) diff --git a/tests/ci/test/test_history_gate.py b/tests/ci/test/test_history_gate.py index 1857d4e471..125388170b 100644 --- a/tests/ci/test/test_history_gate.py +++ b/tests/ci/test/test_history_gate.py @@ -608,6 +608,32 @@ def test_reward_corridor_flags_drop_and_suspicious_jump(tmp_path, store): ) +def test_grad_norm_default_is_a_sanity_band(tmp_path, store): + # grad_norm jitters and is exactly 0 on steps where every GRPO group's rewards + # are uniform, so a value passes up to max(4x the mean, mean + 1.0), down to 0. + test_file = _write_test_file(tmp_path, 'register_ci_gate(metric_key="train/grad_norm")') + for step, values in ((0, [0.2, 0.2]), (1, [2.0, 2.0]), (2, [0.0, 0.0])): + _seed_baseline( + store, + test_file, + metric_key="train/grad_norm", + steps_key=_key("all"), + constraint_key="{}", + step=step, + values=values, + ) + cases = [ + ((0.0, 0.0, 0.0), GateStatus.PASS), + ((1.1, 7.9, 0.9), GateStatus.PASS), + ((1.3, 8.1, 1.1), GateStatus.FAIL), + ] + for currents, status in cases: + series = [[step, value] for step, value in enumerate(currents)] + record = _write_record(tmp_path, {"train/grad_norm": series}, name=f"{currents[0]}.jsonl") + metrics = evaluate_gate(test_file, record, store, executing_suite="stage-c-8-gpu-h100").metrics + assert [(m.baseline_n, m.historical_status) for m in metrics] == [(2, status)] * 3 + + # --- multiple specs, no specs ------------------------------------------------ diff --git a/tests/e2e/fsdp/r3/test_glm47_flash_r3.py b/tests/e2e/fsdp/r3/test_glm47_flash_r3.py index fabca06f57..bf0c77dd05 100644 --- a/tests/e2e/fsdp/r3/test_glm47_flash_r3.py +++ b/tests/e2e/fsdp/r3/test_glm47_flash_r3.py @@ -14,11 +14,7 @@ register_cuda_ci(est_time=800, suite="stage-c-8-gpu-h200", labels=["fsdp", "replay"], hardware=["hopper"], num_gpus=8) register_rocm_ci(est_time=800, suite="nightly-stage-c-8-gpu-mi350", labels=["fsdp", "replay"]) -register_ci_gate( - metric_key="train/grad_norm", - enforce=False, - allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.", -) +register_ci_gate(metric_key="train/grad_norm") register_ci_gate(metric_key="train/ppo_kl") # raw_reward averages 64 samples, so it moves in 1/64 steps and healthy runs land up to 0.11 from the mean. register_ci_gate( diff --git a/tests/e2e/fsdp/r3/test_qwen3_30b_a3b_r3.py b/tests/e2e/fsdp/r3/test_qwen3_30b_a3b_r3.py index 659562f0f4..18c1919e01 100644 --- a/tests/e2e/fsdp/r3/test_qwen3_30b_a3b_r3.py +++ b/tests/e2e/fsdp/r3/test_qwen3_30b_a3b_r3.py @@ -6,11 +6,7 @@ register_cuda_ci(est_time=700, suite="stage-c-8-gpu-h200", labels=["fsdp", "replay"], hardware=["hopper"], num_gpus=8) -register_ci_gate( - metric_key="train/grad_norm", - enforce=False, - allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.", -) +register_ci_gate(metric_key="train/grad_norm") register_ci_gate(metric_key="train/ppo_kl") register_ci_gate(metric_key="rollout/raw_reward") diff --git a/tests/e2e/fsdp/r3/test_qwen3_5_35b_a3b_r3.py b/tests/e2e/fsdp/r3/test_qwen3_5_35b_a3b_r3.py index 7ef617db00..f667c1ba26 100644 --- a/tests/e2e/fsdp/r3/test_qwen3_5_35b_a3b_r3.py +++ b/tests/e2e/fsdp/r3/test_qwen3_5_35b_a3b_r3.py @@ -6,11 +6,7 @@ register_cuda_ci(est_time=1800, suite="stage-c-8-gpu-h200", labels=["fsdp", "replay"], hardware=["hopper"], num_gpus=8) -register_ci_gate( - metric_key="train/grad_norm", - enforce=False, - allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.", -) +register_ci_gate(metric_key="train/grad_norm") register_ci_gate(metric_key="train/ppo_kl") register_ci_gate(metric_key="rollout/raw_reward") diff --git a/tests/e2e/megatron/test_glm47_flash/test_nospec_nor3_fp8_deepep_sgl_dpattn1x4_meg_tp2cp1_bshd.py b/tests/e2e/megatron/test_glm47_flash/test_nospec_nor3_fp8_deepep_sgl_dpattn1x4_meg_tp2cp1_bshd.py index 4df1556486..c81b45aff8 100644 --- a/tests/e2e/megatron/test_glm47_flash/test_nospec_nor3_fp8_deepep_sgl_dpattn1x4_meg_tp2cp1_bshd.py +++ b/tests/e2e/megatron/test_glm47_flash/test_nospec_nor3_fp8_deepep_sgl_dpattn1x4_meg_tp2cp1_bshd.py @@ -15,11 +15,7 @@ # 4x H200: the 8x H200 runners cannot bring NVSHMEM up over IB, which DeepEP low-latency mode needs. register_cuda_ci(est_time=1400, suite="stage-c-4-gpu-h200", labels=["megatron"], hardware=["hopper"], num_gpus=4) -register_ci_gate( - metric_key="train/grad_norm", - enforce=False, - allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.", -) +register_ci_gate(metric_key="train/grad_norm") register_ci_gate(metric_key="train/ppo_kl") register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff") register_ci_gate(metric_key="train/train_rollout_kl") diff --git a/tests/e2e/megatron/test_glm47_flash/test_nospec_r3_bf16_deepep_sgl_dpattn2x2_meg_tp2cp2_bshd.py b/tests/e2e/megatron/test_glm47_flash/test_nospec_r3_bf16_deepep_sgl_dpattn2x2_meg_tp2cp2_bshd.py index 4697f7dd42..22d3be779c 100644 --- a/tests/e2e/megatron/test_glm47_flash/test_nospec_r3_bf16_deepep_sgl_dpattn2x2_meg_tp2cp2_bshd.py +++ b/tests/e2e/megatron/test_glm47_flash/test_nospec_r3_bf16_deepep_sgl_dpattn2x2_meg_tp2cp2_bshd.py @@ -18,11 +18,7 @@ est_time=1500, suite="stage-c-4-gpu-h200", labels=["megatron", "replay"], hardware=["hopper"], num_gpus=4 ) -register_ci_gate( - metric_key="train/grad_norm", - enforce=False, - allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.", -) +register_ci_gate(metric_key="train/grad_norm") register_ci_gate(metric_key="train/ppo_kl") register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff") register_ci_gate(metric_key="train/train_rollout_kl") diff --git a/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_nor3_fp8_sgl_tp2_meg_tp1cp1_fullyasync.py b/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_nor3_fp8_sgl_tp2_meg_tp1cp1_fullyasync.py index 7e17d11048..2430957a42 100644 --- a/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_nor3_fp8_sgl_tp2_meg_tp1cp1_fullyasync.py +++ b/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_nor3_fp8_sgl_tp2_meg_tp1cp1_fullyasync.py @@ -19,11 +19,7 @@ num_gpus=8, ) -register_ci_gate( - metric_key="train/grad_norm", - enforce=False, - allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.", -) +register_ci_gate(metric_key="train/grad_norm") register_ci_gate(metric_key="train/ppo_kl") register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff") register_ci_gate(metric_key="train/train_rollout_kl") diff --git a/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_r3_bf16_sgl_tp2_meg_tp1cp2_disagg.py b/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_r3_bf16_sgl_tp2_meg_tp1cp2_disagg.py index 528e5703ab..97eef9e5f2 100644 --- a/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_r3_bf16_sgl_tp2_meg_tp1cp2_disagg.py +++ b/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_r3_bf16_sgl_tp2_meg_tp1cp2_disagg.py @@ -12,11 +12,7 @@ num_gpus=4, ) -register_ci_gate( - metric_key="train/grad_norm", - enforce=False, - allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.", -) +register_ci_gate(metric_key="train/grad_norm") register_ci_gate(metric_key="train/ppo_kl") register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff") register_ci_gate(metric_key="train/train_rollout_kl") diff --git a/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_r3_bf16_sgl_tp4_meg_tp2pp2.py b/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_r3_bf16_sgl_tp4_meg_tp2pp2.py index 830f761dfb..cbde0ea5a3 100644 --- a/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_r3_bf16_sgl_tp4_meg_tp2pp2.py +++ b/tests/e2e/megatron/test_glm47_flash/test_spec_mtptrain_r3_bf16_sgl_tp4_meg_tp2pp2.py @@ -9,11 +9,7 @@ ) register_rocm_ci(est_time=1500, suite="nightly-stage-c-4-gpu-mi350", labels=["megatron"]) -register_ci_gate( - metric_key="train/grad_norm", - enforce=False, - allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.", -) +register_ci_gate(metric_key="train/grad_norm") register_ci_gate(metric_key="train/ppo_kl") register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff") register_ci_gate(metric_key="train/train_rollout_kl") diff --git a/tests/e2e/megatron/test_qwen3_30B_A3B/test_deepep_fp8.py b/tests/e2e/megatron/test_qwen3_30B_A3B/test_deepep_fp8.py index 907051d03a..e759ebe95c 100644 --- a/tests/e2e/megatron/test_qwen3_30B_A3B/test_deepep_fp8.py +++ b/tests/e2e/megatron/test_qwen3_30B_A3B/test_deepep_fp8.py @@ -8,11 +8,7 @@ est_time=1500, suite="stage-c-4-gpu-h200", labels=["megatron"], hardware=["hopper", "blackwell"], num_gpus=4 ) -register_ci_gate( - metric_key="train/grad_norm", - enforce=False, - allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.", -) +register_ci_gate(metric_key="train/grad_norm") register_ci_gate(metric_key="train/ppo_kl") register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff") register_ci_gate(metric_key="train/train_rollout_kl") diff --git a/tests/e2e/megatron/test_qwen3_30B_A3B/test_deepep_fp8_bridge.py b/tests/e2e/megatron/test_qwen3_30B_A3B/test_deepep_fp8_bridge.py index c0c8b6931b..3ff7e43c41 100644 --- a/tests/e2e/megatron/test_qwen3_30B_A3B/test_deepep_fp8_bridge.py +++ b/tests/e2e/megatron/test_qwen3_30B_A3B/test_deepep_fp8_bridge.py @@ -9,11 +9,7 @@ est_time=1300, suite="stage-c-4-gpu-h200", labels=["megatron"], hardware=["hopper", "blackwell"], num_gpus=4 ) -register_ci_gate( - metric_key="train/grad_norm", - enforce=False, - allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.", -) +register_ci_gate(metric_key="train/grad_norm") register_ci_gate(metric_key="train/ppo_kl") register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff") register_ci_gate(metric_key="train/train_rollout_kl") diff --git a/tests/e2e/megatron/test_qwen3_30B_A3B/test_disagg_broadcast.py b/tests/e2e/megatron/test_qwen3_30B_A3B/test_disagg_broadcast.py index 41b5220b35..7d2fc980fe 100644 --- a/tests/e2e/megatron/test_qwen3_30B_A3B/test_disagg_broadcast.py +++ b/tests/e2e/megatron/test_qwen3_30B_A3B/test_disagg_broadcast.py @@ -13,11 +13,7 @@ ) register_rocm_ci(est_time=900, suite="nightly-stage-c-4-gpu-mi350", labels=["megatron", "weight-update"]) -register_ci_gate( - metric_key="train/grad_norm", - enforce=False, - allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.", -) +register_ci_gate(metric_key="train/grad_norm") register_ci_gate(metric_key="train/ppo_kl") register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff") register_ci_gate(metric_key="train/train_rollout_kl") diff --git a/tests/e2e/megatron/test_qwen3_30B_A3B/test_dp_attention.py b/tests/e2e/megatron/test_qwen3_30B_A3B/test_dp_attention.py index d57dd487a8..71133c99ae 100644 --- a/tests/e2e/megatron/test_qwen3_30B_A3B/test_dp_attention.py +++ b/tests/e2e/megatron/test_qwen3_30B_A3B/test_dp_attention.py @@ -21,11 +21,7 @@ ) register_rocm_ci(est_time=800, suite="nightly-stage-c-4-gpu-mi350", labels=["megatron", "short"]) -register_ci_gate( - metric_key="train/grad_norm", - enforce=False, - allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.", -) +register_ci_gate(metric_key="train/grad_norm") register_ci_gate(metric_key="train/ppo_kl") register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff") register_ci_gate(metric_key="train/train_rollout_kl") diff --git a/tests/e2e/megatron/test_qwen3_30B_A3B/test_int4_rollout.py b/tests/e2e/megatron/test_qwen3_30B_A3B/test_int4_rollout.py index 2a3da0f51e..2c6fdf1d0e 100644 --- a/tests/e2e/megatron/test_qwen3_30B_A3B/test_int4_rollout.py +++ b/tests/e2e/megatron/test_qwen3_30B_A3B/test_int4_rollout.py @@ -8,11 +8,7 @@ est_time=1700, suite="stage-c-4-gpu-h200", labels=["megatron"], hardware=["hopper", "blackwell"], num_gpus=4 ) -register_ci_gate( - metric_key="train/grad_norm", - enforce=False, - allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.", -) +register_ci_gate(metric_key="train/grad_norm") register_ci_gate(metric_key="train/ppo_kl") register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff") register_ci_gate(metric_key="train/train_rollout_kl") diff --git a/tests/e2e/megatron/test_qwen3_30B_A3B/test_nospec_nor3_bf16_sgl_tp4_meg_tp2cp2.py b/tests/e2e/megatron/test_qwen3_30B_A3B/test_nospec_nor3_bf16_sgl_tp4_meg_tp2cp2.py index 2413c512dc..54bac9591a 100644 --- a/tests/e2e/megatron/test_qwen3_30B_A3B/test_nospec_nor3_bf16_sgl_tp4_meg_tp2cp2.py +++ b/tests/e2e/megatron/test_qwen3_30B_A3B/test_nospec_nor3_bf16_sgl_tp4_meg_tp2cp2.py @@ -14,11 +14,7 @@ ) register_rocm_ci(est_time=1000, suite="nightly-stage-c-4-gpu-mi350", labels=["megatron"]) -register_ci_gate( - metric_key="train/grad_norm", - enforce=False, - allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.", -) +register_ci_gate(metric_key="train/grad_norm") register_ci_gate(metric_key="train/ppo_kl") register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff") register_ci_gate(metric_key="train/train_rollout_kl") diff --git a/tests/e2e/megatron/test_qwen3_30B_A3B/test_r3_deepep.py b/tests/e2e/megatron/test_qwen3_30B_A3B/test_r3_deepep.py index 339bf9b84e..bb36f9e02e 100644 --- a/tests/e2e/megatron/test_qwen3_30B_A3B/test_r3_deepep.py +++ b/tests/e2e/megatron/test_qwen3_30B_A3B/test_r3_deepep.py @@ -14,11 +14,7 @@ num_gpus=4, ) -register_ci_gate( - metric_key="train/grad_norm", - enforce=False, - allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.", -) +register_ci_gate(metric_key="train/grad_norm") register_ci_gate(metric_key="train/ppo_kl") register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff") register_ci_gate(metric_key="train/train_rollout_kl") diff --git a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_deepep_sgl_dpattn2x2_meg_tp1cp2.py b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_deepep_sgl_dpattn2x2_meg_tp1cp2.py index eb17017603..ed072c42a9 100644 --- a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_deepep_sgl_dpattn2x2_meg_tp1cp2.py +++ b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_deepep_sgl_dpattn2x2_meg_tp1cp2.py @@ -15,11 +15,7 @@ est_time=1500, suite="stage-c-4-gpu-h200", labels=["megatron", "qwen35"], hardware=["hopper"], num_gpus=4 ) -register_ci_gate( - metric_key="train/grad_norm", - enforce=False, - allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.", -) +register_ci_gate(metric_key="train/grad_norm") register_ci_gate(metric_key="train/ppo_kl") register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff") register_ci_gate(metric_key="train/train_rollout_kl") diff --git a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_sgl_dpattn2x2_meg_tp2pp2.py b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_sgl_dpattn2x2_meg_tp2pp2.py index 2867d8589c..34f3835519 100644 --- a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_sgl_dpattn2x2_meg_tp2pp2.py +++ b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_nor3_bf16_sgl_dpattn2x2_meg_tp2pp2.py @@ -18,11 +18,7 @@ ) register_rocm_ci(est_time=1300, suite="nightly-stage-c-4-gpu-mi350", labels=["megatron", "qwen35"]) -register_ci_gate( - metric_key="train/grad_norm", - enforce=False, - allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.", -) +register_ci_gate(metric_key="train/grad_norm") register_ci_gate(metric_key="train/ppo_kl") register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff") register_ci_gate(metric_key="train/train_rollout_kl") diff --git a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_r3_bf16_sgl_dpattn1x4_meg_tp1cp1_fullyasync.py b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_r3_bf16_sgl_dpattn1x4_meg_tp1cp1_fullyasync.py index 498a1d7059..127b1675ca 100644 --- a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_r3_bf16_sgl_dpattn1x4_meg_tp1cp1_fullyasync.py +++ b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_nospec_r3_bf16_sgl_dpattn1x4_meg_tp1cp1_fullyasync.py @@ -27,11 +27,7 @@ labels=["megatron", "qwen35", "weight-update", "fully-async", "replay"], ) -register_ci_gate( - metric_key="train/grad_norm", - enforce=False, - allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.", -) +register_ci_gate(metric_key="train/grad_norm") register_ci_gate(metric_key="train/ppo_kl") register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff") register_ci_gate(metric_key="train/train_rollout_kl") diff --git a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_mtptrain_r3_fp8_deepep_sgl_dpattn1x4_meg_tp2cp1.py b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_mtptrain_r3_fp8_deepep_sgl_dpattn1x4_meg_tp2cp1.py index 71ca9ea043..28e0d918fd 100644 --- a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_mtptrain_r3_fp8_deepep_sgl_dpattn1x4_meg_tp2cp1.py +++ b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_mtptrain_r3_fp8_deepep_sgl_dpattn1x4_meg_tp2cp1.py @@ -18,11 +18,7 @@ est_time=2200, suite="stage-c-4-gpu-h200", labels=["megatron", "qwen35", "replay"], hardware=["hopper"], num_gpus=4 ) -register_ci_gate( - metric_key="train/grad_norm", - enforce=False, - allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.", -) +register_ci_gate(metric_key="train/grad_norm") register_ci_gate(metric_key="train/ppo_kl") register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff") register_ci_gate(metric_key="train/train_rollout_kl") diff --git a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_nor3_fp8_sgl_tp4_meg_tp2cp2.py b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_nor3_fp8_sgl_tp4_meg_tp2cp2.py index 25156a3195..45c2c71d74 100644 --- a/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_nor3_fp8_sgl_tp4_meg_tp2cp2.py +++ b/tests/e2e/megatron/test_qwen3_5_35B_A3B/test_spec_nor3_fp8_sgl_tp4_meg_tp2cp2.py @@ -27,11 +27,7 @@ ) register_rocm_ci(est_time=1600, suite="nightly-stage-c-4-gpu-mi350", labels=["megatron", "qwen35"]) -register_ci_gate( - metric_key="train/grad_norm", - enforce=False, - allowlist_reason="Exact-zero steps (every GRPO group's rewards uniform) make per-step grad_norm bimodal.", -) +register_ci_gate(metric_key="train/grad_norm") register_ci_gate(metric_key="train/ppo_kl") register_ci_gate(metric_key="train/train_rollout_logprob_abs_diff") register_ci_gate(metric_key="train/train_rollout_kl")