diff --git a/docs/eval-agentx-procedures.md b/docs/eval-agentx-procedures.md index 469bd4aafd..456b35207a 100644 --- a/docs/eval-agentx-procedures.md +++ b/docs/eval-agentx-procedures.md @@ -11,6 +11,16 @@ Use this page to add and run graded evals, operate AgentX trace replays, preserv ## 1. Pick the correct execution mode +For a throughput-only PR sweep, set `no-evals: true` on its +`perf-changelog.yaml` entries and use a normal primary sweep label, including +`full-sweep-enabled`. This skips all eval job families for those entries without +changing benchmark duration or Prometheus artifacts. The flag defaults to false +and is retained in changelog metadata. Another entry requesting the same config +can still select its evals; mark every applicable entry to suppress them entirely. +Combining `no-evals` with `all-evals`, `evals-only`, or `eval-min-prefill-ep` +on the entry, or with either eval PR modifier, is rejected. Such a run provides +throughput evidence, not model-evaluation evidence. + There are two distinct layers: the matrix generator decides **which jobs exist**, while runtime variables decide **what a launched job does**. | Need | Generator/workflow mode | Runtime behavior | diff --git a/docs/eval-agentx-procedures_zh.md b/docs/eval-agentx-procedures_zh.md index 501344bf25..215a4df4af 100644 --- a/docs/eval-agentx-procedures_zh.md +++ b/docs/eval-agentx-procedures_zh.md @@ -11,6 +11,14 @@ ## 1. 选择正确的执行模式 +若 PR sweep 只需测试吞吐量,请在相应的 `perf-changelog.yaml` 条目中设置 +`no-evals: true`,并使用常规主要标签(包括 `full-sweep-enabled`)。这会跳过 +这些条目的所有 eval 作业,不改变 benchmark 时长或 Prometheus 产物。 +该选项默认为 false,且保留在 changelog 元数据中。其他条目仍可为同一配置 +选择 eval;若需完全禁用,请在所有相关条目中设置该选项。条目中不能同时使用 +`all-evals`、`evals-only` 或 `eval-min-prefill-ep`,PR 也不能同时使用两个 eval +modifier 中的任意一个。这类运行提供吞吐量证据,不提供模型评估证据。 + 这里有两个不同层次:矩阵生成器决定**存在哪些作业**,运行时变量决定**已启动作业执行什么操作**。 | 需求 | 生成器/工作流模式 | 运行时行为 | diff --git a/infx/matrix/validation.py b/infx/matrix/validation.py index 7014e54298..a3614b0bc1 100644 --- a/infx/matrix/validation.py +++ b/infx/matrix/validation.py @@ -1031,6 +1031,7 @@ class ChangelogEntry(BaseModel): pr_link: str = Field(alias="pr-link") evals_only: bool = Field(alias="evals-only", default=False) all_evals: bool = Field(alias="all-evals", default=False) + no_evals: bool = Field(alias="no-evals", default=False) append_only: bool = Field( alias="append-only", default=False, @@ -1054,6 +1055,10 @@ class ChangelogEntry(BaseModel): @model_validator(mode="after") def validate_append_only_mode(self): """Append-only entries are throughput deltas, never eval-only requests.""" + if self.no_evals and ( + self.evals_only or self.all_evals or self.eval_min_prefill_ep is not None + ): + raise ValueError("no-evals cannot be combined with eval selection fields") if self.append_only and ( self.evals_only or self.all_evals or self.eval_min_prefill_ep is not None ): diff --git a/utils/process_changelog.py b/utils/process_changelog.py index cd61535463..8d9082858b 100644 --- a/utils/process_changelog.py +++ b/utils/process_changelog.py @@ -430,6 +430,10 @@ def main(): raise ValueError("No valid YAML entries found in the changelog additions.") parsed_entries = [ChangelogEntry.model_validate(entry) for entry in changelog_data] + if any(entry.no_evals for entry in parsed_entries) and ( + args.all_evals or args.evals_only + ): + raise ValueError("no-evals entries cannot use all-evals or evals-only modifiers") has_append_only = any(entry.append_only for entry in parsed_entries) if has_append_only and not all(entry.append_only for entry in parsed_entries): raise ValueError( @@ -522,7 +526,7 @@ def main(): head_results = append_only_delta(base_results, head_results) all_benchmark_results.extend(head_results) - if entry.append_only: + if entry.append_only or entry.no_evals: continue eval_groups = group_unseen_scenarios( diff --git a/utils/test_process_changelog.py b/utils/test_process_changelog.py index dedec90787..407502c641 100644 --- a/utils/test_process_changelog.py +++ b/utils/test_process_changelog.py @@ -1011,6 +1011,43 @@ def generate(command, **kwargs): return run +@pytest.mark.parametrize("skip", [False, True]) +def test_no_evals_preserves_throughput_and_metadata(changelog_run, skip): + output, commands = changelog_run( + [{"no-evals": skip}], generated=lambda _: [_fixed_matrix_row(8)], + ) + assert [row["conc"] for row in output["single_node"]["8k1k"]] == [8] + assert len(output["evals"]) == (0 if skip else 1) + assert output["agentic_evals"] == [] + assert output["multinode_evals"] == [] + assert output["multinode_agentic_evals"] == [] + assert len(commands) == (1 if skip else 2) + assert output["changelog_metadata"]["entries"][0]["no-evals"] is skip + + +def test_no_evals_does_not_suppress_another_entry(changelog_run): + _, commands = changelog_run([ + {"config-keys": ["config-a"], "no-evals": True}, + {"config-keys": ["config-b"]}, + ]) + eval_commands = [command for command in commands if "--evals-only" in command] + assert len(eval_commands) == 1 + assert eval_commands[0][eval_commands[0].index("--config-keys") + 1] == "config-b" + + +@pytest.mark.parametrize("flags", [{"evals-only": True}, {"all-evals": True}, + {"eval-min-prefill-ep": 2}]) +def test_no_evals_rejects_conflicting_entry_options(changelog_run, flags): + with pytest.raises(ValueError, match="no-evals cannot be combined"): + changelog_run([{"no-evals": True, **flags}]) + + +@pytest.mark.parametrize("flag", ["--all-evals", "--evals-only"]) +def test_no_evals_rejects_conflicting_pr_modifiers(changelog_run, flag): + with pytest.raises(ValueError, match="no-evals entries cannot use"): + changelog_run([{"no-evals": True}], [flag]) + + # The table is the contract: CLI expansion preserves throughput, whereas an # entry requesting all evals is eval-only. Trimming affects throughput alone. @pytest.mark.parametrize("cli_flags,expected_modes", [