feat: support no-evals in perf changelog entries

docs/eval-agentx-procedures.md

@@ -11,6 +11,16 @@ Use this page to add and run graded evals, operate AgentX trace replays, preserv
 
 ## 1. Pick the correct execution mode
 
+For a throughput-only PR sweep, set `no-evals: true` on its
+`perf-changelog.yaml` entries and use a normal primary sweep label, including
+`full-sweep-enabled`. This skips all eval job families for those entries without
+changing benchmark duration or Prometheus artifacts. The flag defaults to false
+and is retained in changelog metadata. Another entry requesting the same config
+can still select its evals; mark every applicable entry to suppress them entirely.
+Combining `no-evals` with `all-evals`, `evals-only`, or `eval-min-prefill-ep`
+on the entry, or with either eval PR modifier, is rejected. Such a run provides
+throughput evidence, not model-evaluation evidence.
+
 There are two distinct layers: the matrix generator decides **which jobs exist**, while runtime variables decide **what a launched job does**.
 
 | Need | Generator/workflow mode | Runtime behavior |

docs/eval-agentx-procedures_zh.md

@@ -11,6 +11,14 @@
 
 ## 1. 选择正确的执行模式
 
+若 PR sweep 只需测试吞吐量，请在相应的 `perf-changelog.yaml` 条目中设置
+`no-evals: true`，并使用常规主要标签（包括 `full-sweep-enabled`）。这会跳过
+这些条目的所有 eval 作业，不改变 benchmark 时长或 Prometheus 产物。
+该选项默认为 false，且保留在 changelog 元数据中。其他条目仍可为同一配置
+选择 eval；若需完全禁用，请在所有相关条目中设置该选项。条目中不能同时使用
+`all-evals`、`evals-only` 或 `eval-min-prefill-ep`，PR 也不能同时使用两个 eval
+modifier 中的任意一个。这类运行提供吞吐量证据，不提供模型评估证据。
+
 这里有两个不同层次：矩阵生成器决定**存在哪些作业**，运行时变量决定**已启动作业执行什么操作**。
 
 | 需求 | 生成器/工作流模式 | 运行时行为 |

infx/matrix/validation.py

@@ -1031,6 +1031,7 @@ class ChangelogEntry(BaseModel):
     pr_link: str = Field(alias="pr-link")
     evals_only: bool = Field(alias="evals-only", default=False)
     all_evals: bool = Field(alias="all-evals", default=False)
+    no_evals: bool = Field(alias="no-evals", default=False)
     append_only: bool = Field(
         alias="append-only",
         default=False,
@@ -1054,6 +1055,10 @@ class ChangelogEntry(BaseModel):
     @model_validator(mode="after")
     def validate_append_only_mode(self):
         """Append-only entries are throughput deltas, never eval-only requests."""
+        if self.no_evals and (
+            self.evals_only or self.all_evals or self.eval_min_prefill_ep is not None
+        ):
+            raise ValueError("no-evals cannot be combined with eval selection fields")
         if self.append_only and (
             self.evals_only or self.all_evals or self.eval_min_prefill_ep is not None
         ):

utils/process_changelog.py

@@ -430,6 +430,10 @@ def main():
         raise ValueError("No valid YAML entries found in the changelog additions.")
 
     parsed_entries = [ChangelogEntry.model_validate(entry) for entry in changelog_data]
+    if any(entry.no_evals for entry in parsed_entries) and (
+        args.all_evals or args.evals_only
+    ):
+        raise ValueError("no-evals entries cannot use all-evals or evals-only modifiers")
     has_append_only = any(entry.append_only for entry in parsed_entries)
     if has_append_only and not all(entry.append_only for entry in parsed_entries):
         raise ValueError(
@@ -522,7 +526,7 @@ def main():
                     head_results = append_only_delta(base_results, head_results)
                 all_benchmark_results.extend(head_results)
 
-        if entry.append_only:
+        if entry.append_only or entry.no_evals:
             continue
 
         eval_groups = group_unseen_scenarios(

utils/test_process_changelog.py

@@ -1011,6 +1011,43 @@ def generate(command, **kwargs):
     return run
 
 
+@pytest.mark.parametrize("skip", [False, True])
+def test_no_evals_preserves_throughput_and_metadata(changelog_run, skip):
+    output, commands = changelog_run(
+        [{"no-evals": skip}], generated=lambda _: [_fixed_matrix_row(8)],
+    )
+    assert [row["conc"] for row in output["single_node"]["8k1k"]] == [8]
+    assert len(output["evals"]) == (0 if skip else 1)
+    assert output["agentic_evals"] == []
+    assert output["multinode_evals"] == []
+    assert output["multinode_agentic_evals"] == []
+    assert len(commands) == (1 if skip else 2)
+    assert output["changelog_metadata"]["entries"][0]["no-evals"] is skip
+
+
+def test_no_evals_does_not_suppress_another_entry(changelog_run):
+    _, commands = changelog_run([
+        {"config-keys": ["config-a"], "no-evals": True},
+        {"config-keys": ["config-b"]},
+    ])
+    eval_commands = [command for command in commands if "--evals-only" in command]
+    assert len(eval_commands) == 1
+    assert eval_commands[0][eval_commands[0].index("--config-keys") + 1] == "config-b"
+
+
+@pytest.mark.parametrize("flags", [{"evals-only": True}, {"all-evals": True},
+                                   {"eval-min-prefill-ep": 2}])
+def test_no_evals_rejects_conflicting_entry_options(changelog_run, flags):
+    with pytest.raises(ValueError, match="no-evals cannot be combined"):
+        changelog_run([{"no-evals": True, **flags}])
+
+
+@pytest.mark.parametrize("flag", ["--all-evals", "--evals-only"])
+def test_no_evals_rejects_conflicting_pr_modifiers(changelog_run, flag):
+    with pytest.raises(ValueError, match="no-evals entries cannot use"):
+        changelog_run([{"no-evals": True}], [flag])
+
+
 # The table is the contract: CLI expansion preserves throughput, whereas an
 # entry requesting all evals is eval-only. Trimming affects throughput alone.
 @pytest.mark.parametrize("cli_flags,expected_modes", [