From 2ca8e89796aa6816c693303d5ac69986364757f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Sch=C3=BCller?= Date: Mon, 10 Aug 2026 15:24:10 +0200 Subject: [PATCH 1/3] feat(experimentalist): stop a run once the objective is reached MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The loop had two ways to stop: the round budget, and a judgement that progress had stalled. Neither notices success. A round that goes 0.333 -> 1.000 is the clearest possible case of *not* stagnating, so a solved run kept buying rounds that could only match what it already had -- and a baseline that already met the goal paid for the whole budget before returning itself. Adds an optional `target` to MetricTarget: the value at which that objective is satisfied, in the metric's own units. When every targeted objective is met, the run stops. Unset, nothing changes -- which is the only safe default, because metrics are not required to be normalized and there is no value that means "as good as possible" for an arbitrary one. Three properties worth stating, because each was a decision: Not gated by `disable_convergence_check`. That flag turns off an *inference* about stagnation, which is worth disabling when the signal is unreliable. This is a measurement against a threshold the caller stated. Anyone who does not want the run to stop here simply does not set a target. Checked at the top of every round including the first. The baseline is scored before the loop begins, so a baseline that already meets the targets ends the run without a single round being paid for. Only candidates that could win are considered -- survivors and the round-0 baseline, mirroring finalization. A killed candidate meeting the target would otherwise end the run in favour of a winner that never reaches it. Insight-authored objectives carry no target, so Mode 1 is unaffected: an LLM-invented metric has no known satisfied value. test_metric_contract now pins that, alongside the `target` a demoted objective keeps when it becomes a guardrail. Signed-off-by: Christian Schüller --- .../src/nemo_experimentalist_plugin/config.py | 20 +++ .../experimentalist/components/terminator.py | 42 ++++++ .../experimentalist/test_objective_reached.py | 138 ++++++++++++++++++ .../tests/test_metric_contract.py | 8 +- 4 files changed, 206 insertions(+), 2 deletions(-) create mode 100644 plugins/nemo-experimentalist/tests/experimentalist/test_objective_reached.py diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/config.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/config.py index 440c1758972..e5dc55da362 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/config.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/config.py @@ -57,6 +57,26 @@ class MetricTarget(BaseModel): direction: Literal["maximize", "minimize"] = Field( description="Whether higher or lower values are better for this target." ) + target: float | None = Field( + default=None, + description=( + "Value at which this objective counts as satisfied, in the metric's own " + "units. When every targeted objective is met the run stops, so a solved " + "problem stops paying for rounds. Unset means no such stop: metrics are not " + "required to be normalized, so there is no value that means 'as good as " + "possible' for an arbitrary one." + ), + ) + + def is_satisfied_by(self, value: float | None) -> bool: + """Whether *value* meets this target. False when either side is absent. + + A missing measurement is not evidence of success, and a target that was never + configured must not end a run. + """ + if value is None or self.target is None: + return False + return value >= self.target if self.direction == "maximize" else value <= self.target def pareto_objectives(metrics: dict[str, float], objective_function: list[MetricTarget]) -> dict[str, float]: diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/terminator.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/terminator.py index 5e12ceb3532..09c6bc40c0a 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/terminator.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/terminator.py @@ -77,6 +77,9 @@ async def run( Returns: A :class:`TerminationDecision`. """ + reached = self.assess_objective_reached(evolution_tree=evolution_tree, config=config) + if reached.stop: + return reached budget = self.assess_round_budget(round_num=round_num, config=config) if budget.stop: return budget @@ -86,6 +89,45 @@ async def run( config=config, ) + @hidden + def assess_objective_reached( + self, + *, + evolution_tree: EvolutionTree, + config: EvolutionaryOptimizerConfig, + ) -> TerminationDecision: + """Stop when a candidate that could win already satisfies every targeted objective. + + Distinct from convergence, which asks whether progress has *stalled*. This asks + whether there is any progress left worth making, and answers it from the numbers + rather than from a judgement about the analysis text. A run that has solved the + problem otherwise keeps buying rounds until its budget runs out, and each one can + only match what it already has. + + Deliberately **not** gated by ``disable_convergence_check``. That flag turns off + an inference about stagnation, which is a judgement call worth disabling when the + signal is unreliable. This is a measurement against a threshold the caller stated, + and there is no reason to want it off while a target is configured -- if you do + not want the run to stop here, do not set a target. + + Consulted at the top of every round including the first, so a baseline that + already meets the targets ends the run without a single round being paid for. + + Only nodes that could actually win are considered -- survivors and the round-0 + baseline, mirroring finalization. A killed candidate meeting the target would end + the run in favour of a winner that never reaches it. + """ + targets = [target for target in config.objective_function if target.target is not None] + if not targets: + return TerminationDecision(stop=False) + for node in evolution_tree.nodes.values(): + if not node.val_reward or not (node.is_survivor or node.round == 0): + continue + if all(target.is_satisfied_by(node.val_reward.get(target.name)) for target in targets): + summary = ", ".join(f"{target.name}={node.val_reward.get(target.name)}" for target in targets) + return TerminationDecision(stop=True, reason=f"objective reached by {node.label} ({summary})") + return TerminationDecision(stop=False) + @hidden async def assess_convergence( self, diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_objective_reached.py b/plugins/nemo-experimentalist/tests/experimentalist/test_objective_reached.py new file mode 100644 index 00000000000..e1ec72a5d10 --- /dev/null +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_objective_reached.py @@ -0,0 +1,138 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pin when a solved objective ends a run. + +Before this, the only stop conditions were the round budget and a stagnation +judgement. Neither notices success: a round that goes 0.333 -> 1.000 is the +clearest possible case of *not* stagnating, so a solved run kept buying rounds +that could only match what it already had. + +The rule: when every objective carrying a ``target`` is satisfied by a candidate +that could win, stop. Absent targets, nothing changes. +""" + +from __future__ import annotations + +import pytest +from nemo_experimentalist_plugin.config import EvolutionaryOptimizerConfig, MetricTarget +from nemo_experimentalist_plugin.entities import Candidate, RewardRecord +from nemo_experimentalist_plugin.experimentalist.components.models import EvolutionNode, EvolutionTree +from nemo_experimentalist_plugin.experimentalist.components.terminator import Terminator + +SOLVED = [MetricTarget(name="reward", direction="maximize", target=1.0)] +UNTARGETED = [MetricTarget(name="reward", direction="maximize")] + + +def _node(label: str, round_: int, killed: int | None = None, **metrics: float) -> EvolutionNode: + candidate = Candidate( + run_id="run-1", + label=label, + round=round_, + optimization="baseline" if round_ == 0 else "some change", + rewards={"validation": RewardRecord(metrics=dict(metrics))}, + ) + candidate.killed_round = killed + return EvolutionNode(candidate=candidate) + + +def _tree(*nodes: EvolutionNode) -> EvolutionTree: + tree = EvolutionTree() + for node in nodes: + tree.add(node.candidate) + return tree + + +def _assess(tree: EvolutionTree, objectives: list[MetricTarget], **overrides: object): + config = EvolutionaryOptimizerConfig.model_validate( + {"objective_function": [target.model_dump() for target in objectives], **overrides} + ) + return Terminator.assess_objective_reached(Terminator, evolution_tree=tree, config=config) # type: ignore[arg-type] + + +def test_a_solved_objective_stops_the_run() -> None: + decision = _assess(_tree(_node("agent-0", 0, reward=0.333), _node("agent-1", 1, reward=1.0)), SOLVED) + assert decision.stop + assert "agent-1" in decision.reason + + +def test_an_unsolved_objective_does_not_stop_the_run() -> None: + decision = _assess(_tree(_node("agent-0", 0, reward=0.333), _node("agent-1", 1, reward=0.667)), SOLVED) + assert not decision.stop + + +def test_no_target_configured_never_stops() -> None: + """Absent an explicit target there is no value that means 'as good as possible'.""" + decision = _assess(_tree(_node("agent-0", 0, reward=1.0), _node("agent-1", 1, reward=1.0)), UNTARGETED) + assert not decision.stop + + +def test_a_baseline_that_already_meets_the_target_stops_before_any_round() -> None: + """The terminator is consulted at the top of round 1, so nothing is paid for.""" + decision = _assess(_tree(_node("agent-0", 0, reward=1.0)), SOLVED) + assert decision.stop + assert "agent-0" in decision.reason + + +def test_every_targeted_objective_must_be_met() -> None: + objectives = [ + MetricTarget(name="reward", direction="maximize", target=1.0), + MetricTarget(name="shape_ok", direction="maximize", target=1.0), + ] + partial = _assess(_tree(_node("agent-1", 1, reward=1.0, shape_ok=0.5)), objectives) + assert not partial.stop + both = _assess(_tree(_node("agent-1", 1, reward=1.0, shape_ok=1.0)), objectives) + assert both.stop + + +def test_an_untargeted_objective_does_not_block_the_stop() -> None: + """Only targeted objectives are judged; an untargeted one is not an obstacle.""" + objectives = [ + MetricTarget(name="reward", direction="maximize", target=1.0), + MetricTarget(name="coverage", direction="maximize"), + ] + decision = _assess(_tree(_node("agent-1", 1, reward=1.0, coverage=0.1)), objectives) + assert decision.stop + + +def test_minimized_objective_uses_the_opposite_comparison() -> None: + objectives = [MetricTarget(name="cost", direction="minimize", target=0.2)] + assert _assess(_tree(_node("agent-1", 1, cost=0.1)), objectives).stop + assert not _assess(_tree(_node("agent-1", 1, cost=0.3)), objectives).stop + + +def test_a_missing_measurement_is_not_success() -> None: + """The metric was never reported; absence must not be read as meeting the target.""" + decision = _assess(_tree(_node("agent-1", 1, other=1.0)), SOLVED) + assert not decision.stop + + +def test_a_killed_candidate_cannot_end_the_run() -> None: + """It cannot win, so stopping on it would ship a winner that never met the target.""" + decision = _assess( + _tree(_node("agent-0", 0, reward=0.333), _node("agent-1", 1, killed=1, reward=1.0)), + SOLVED, + ) + assert not decision.stop + + +def test_a_killed_baseline_still_counts() -> None: + """Finalization re-admits the baseline, so it remains a candidate for winning.""" + decision = _assess(_tree(_node("agent-0", 0, killed=1, reward=1.0)), SOLVED) + assert decision.stop + + +def test_unscored_nodes_are_ignored() -> None: + decision = _assess(_tree(_node("agent-0", 0, reward=0.5), _node("agent-1", 1)), SOLVED) + assert not decision.stop + + +@pytest.mark.parametrize("disabled", [True, False]) +def test_the_stop_is_independent_of_disable_convergence_check(disabled: bool) -> None: + """That flag turns off a judgement about stagnation; this is a stated threshold.""" + decision = _assess( + _tree(_node("agent-1", 1, reward=1.0)), + SOLVED, + disable_convergence_check=disabled, + ) + assert decision.stop diff --git a/plugins/nemo-experimentalist/tests/test_metric_contract.py b/plugins/nemo-experimentalist/tests/test_metric_contract.py index bb01578c819..753967a8690 100644 --- a/plugins/nemo-experimentalist/tests/test_metric_contract.py +++ b/plugins/nemo-experimentalist/tests/test_metric_contract.py @@ -97,7 +97,11 @@ def test_insight_metrics_become_objectives_and_existing_targets_become_guardrail assert [target.name for target in effective.objective_function] == ["uses_required_tool", "cites_source"] assert all(target.direction == "maximize" for target in effective.objective_function) + # Demoted targets are carried across whole, `target` included. The authored insight + # objectives get none: an LLM-invented metric has no known satisfied value, so a + # Mode 1 run cannot stop early on one and falls back to its round budget. assert [target.model_dump() for target in effective.regression_metrics] == [ - {"name": "cost", "direction": "minimize"}, - {"name": "safety", "direction": "maximize"}, + {"name": "cost", "direction": "minimize", "target": None}, + {"name": "safety", "direction": "maximize", "target": None}, ] + assert all(target.target is None for target in effective.objective_function) From 1e7f9018b4525899ae13a25f1a096cbcc8038015 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Sch=C3=BCller?= Date: Mon, 10 Aug 2026 15:29:04 +0200 Subject: [PATCH 2/3] docs(experimentalist): tighten the assess_objective_reached docstring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Twenty lines restating the commit message. Cut to the four things a caller needs: how it differs from convergence, why `disable_convergence_check` does not gate it, that it runs before the first round, and which candidates are eligible. The reasoning behind each stays in the commit that introduced them. Signed-off-by: Christian Schüller --- .../experimentalist/components/terminator.py | 25 ++++++------------- 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/terminator.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/terminator.py index 09c6bc40c0a..46344d68b96 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/terminator.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/terminator.py @@ -98,24 +98,13 @@ def assess_objective_reached( ) -> TerminationDecision: """Stop when a candidate that could win already satisfies every targeted objective. - Distinct from convergence, which asks whether progress has *stalled*. This asks - whether there is any progress left worth making, and answers it from the numbers - rather than from a judgement about the analysis text. A run that has solved the - problem otherwise keeps buying rounds until its budget runs out, and each one can - only match what it already has. - - Deliberately **not** gated by ``disable_convergence_check``. That flag turns off - an inference about stagnation, which is a judgement call worth disabling when the - signal is unreliable. This is a measurement against a threshold the caller stated, - and there is no reason to want it off while a target is configured -- if you do - not want the run to stop here, do not set a target. - - Consulted at the top of every round including the first, so a baseline that - already meets the targets ends the run without a single round being paid for. - - Only nodes that could actually win are considered -- survivors and the round-0 - baseline, mirroring finalization. A killed candidate meeting the target would end - the run in favour of a winner that never reaches it. + Convergence asks whether progress has stalled; this asks whether any is left to + make. Not gated by ``disable_convergence_check``: that disables a judgement about + stagnation, this is a threshold the caller stated. To keep going, set no target. + + Consulted before every round, so a baseline that already qualifies costs nothing. + Only survivors and the round-0 baseline count, mirroring finalization -- a killed + candidate would end the run in favour of a winner that never reaches the target. """ targets = [target for target in config.objective_function if target.target is not None] if not targets: From f7e66e155248cac19a9770c43dc5b6d491dc282c Mon Sep 17 00:00:00 2001 From: Gaia Di Lorenzo Date: Tue, 11 Aug 2026 16:05:08 +0200 Subject: [PATCH 3/3] fix(experimentalist): accept numeric objective targets Signed-off-by: Gaia Di Lorenzo --- .../experimentalist/components/analyzer.py | 22 +++++----- .../experimentalist/components/proposer.py | 8 ++-- .../components/trace_analyzer.py | 12 ++--- .../tests/test_experimentalist_analyzer.py | 44 ++++++++++++++----- 4 files changed, 55 insertions(+), 31 deletions(-) diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/analyzer.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/analyzer.py index f8316e179a2..566069f7582 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/analyzer.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/analyzer.py @@ -272,8 +272,8 @@ async def select_trials( agent_id: str, dataset: Dataset, evaluation: EvaluationResult, - objective_metrics: list[dict[str, str]], - regression_metrics: list[dict[str, str]], + objective_metrics: list[dict[str, Any]], + regression_metrics: list[dict[str, Any]], ) -> list[TrialSelection]: """Pick which trials to analyze in depth and explain each choice. @@ -351,8 +351,8 @@ async def classify_failures( agent_id: str, diagnoses: list[Diagnostic], trials: Sequence[TrialResult], - objective_metrics: list[dict[str, str]], - regression_metrics: list[dict[str, str]], + objective_metrics: list[dict[str, Any]], + regression_metrics: list[dict[str, Any]], ) -> FailureClassification: """Classify diagnoses into systematic vs. one-off and agent vs. mechanical failures. @@ -388,8 +388,8 @@ async def compare_with_peers( evaluation: EvaluationResult, diagnoses: list[Diagnostic], peer_evaluations: dict[str, EvaluationResult] | None = None, - objective_metrics: list[dict[str, str]] | None = None, - regression_metrics: list[dict[str, str]] | None = None, + objective_metrics: list[dict[str, Any]] | None = None, + regression_metrics: list[dict[str, Any]] | None = None, ) -> PeerComparison: """Compare this agent to peers and return divergent trials and complementary patterns. @@ -513,7 +513,7 @@ def _task_metric_means(self, evaluation: EvaluationResult) -> dict[str, dict[str @staticmethod def _metric_directions( - objective_metrics: list[dict[str, str]], regression_metrics: list[dict[str, str]] + objective_metrics: list[dict[str, Any]], regression_metrics: list[dict[str, Any]] ) -> dict[str, str]: """Return metric directions, defaulting dimensions outside the contract to maximize.""" return {metric["name"]: metric["direction"] for metric in [*objective_metrics, *regression_metrics]} @@ -572,8 +572,8 @@ async def _narrate_peer_comparison( top_divergent: list[dict[str, Any]], complementary_raw: dict[str, dict[str, dict[str, list[str]]]], diagnoses: list[Diagnostic], - objective_metrics: list[dict[str, str]], - regression_metrics: list[dict[str, str]], + objective_metrics: list[dict[str, Any]], + regression_metrics: list[dict[str, Any]], ) -> PeerComparison: """Write the DivergentTrial and ComplementaryPattern narratives from pre-computed data. @@ -652,8 +652,8 @@ async def run( client: AsyncNeMoPlatform | None = None, nmp_workspace: str | None = None, agent_spec: Path | None = None, - objective_metrics: list[dict[str, str]] | None = None, - regression_metrics: list[dict[str, str]] | None = None, + objective_metrics: list[dict[str, Any]] | None = None, + regression_metrics: list[dict[str, Any]] | None = None, ) -> AgentAnalysis: """Run the full analysis pipeline for one agent in one optimization round. diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/proposer.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/proposer.py index 6c1d647d5f0..331bd284590 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/proposer.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/proposer.py @@ -101,8 +101,8 @@ async def run( round_num: int, phase: Literal["exploration", "exploitation"], max_candidates: int, - objective_metrics: list[dict[str, str]], - regression_metrics: list[dict[str, str]], + objective_metrics: list[dict[str, Any]], + regression_metrics: list[dict[str, Any]], ) -> list[Improvement]: """Return up to max_candidates targeted improvement proposals. @@ -242,8 +242,8 @@ async def _run_with_context( cards_index: str, phase: Literal["exploration", "exploitation"], max_candidates: int, - objective_metrics: list[dict[str, str]], - regression_metrics: list[dict[str, str]], + objective_metrics: list[dict[str, Any]], + regression_metrics: list[dict[str, Any]], ) -> list[Improvement]: """Pick up to `max_candidates` targeted improvements grounded in root causes. diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_analyzer.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_analyzer.py index fc2feb5e736..956eefd0571 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_analyzer.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_analyzer.py @@ -194,8 +194,8 @@ async def analyze_trajectory( runtime: DependencyRuntime | None, insight: Insight | None = None, selection_reason: str = "", - objective_metrics: list[dict[str, str]] | None = None, - regression_metrics: list[dict[str, str]] | None = None, + objective_metrics: list[dict[str, Any]] | None = None, + regression_metrics: list[dict[str, Any]] | None = None, ) -> StepAnalysis: """Trace the agent's path and find where it went wrong. @@ -259,8 +259,8 @@ async def diagnose( trace: TraceExplorer, analysis: StepAnalysis, insight: Insight | None = None, - objective_metrics: list[dict[str, str]] | None = None, - regression_metrics: list[dict[str, str]] | None = None, + objective_metrics: list[dict[str, Any]] | None = None, + regression_metrics: list[dict[str, Any]] | None = None, ) -> Diagnostic: """Determine the primary root cause and produce a Diagnostic. @@ -302,8 +302,8 @@ async def run( rationale: Rationale | None = None, insight: Insight | None = None, selection_reason: str = "", - objective_metrics: list[dict[str, str]] | None = None, - regression_metrics: list[dict[str, str]] | None = None, + objective_metrics: list[dict[str, Any]] | None = None, + regression_metrics: list[dict[str, Any]] | None = None, client: AsyncNeMoPlatform | None = None, workspace: str | None = None, ) -> Diagnostic: diff --git a/plugins/nemo-experimentalist/tests/test_experimentalist_analyzer.py b/plugins/nemo-experimentalist/tests/test_experimentalist_analyzer.py index 7bac381067d..7f766e5599f 100644 --- a/plugins/nemo-experimentalist/tests/test_experimentalist_analyzer.py +++ b/plugins/nemo-experimentalist/tests/test_experimentalist_analyzer.py @@ -89,8 +89,8 @@ async def run( rationale: Any = None, insight: Any = None, selection_reason: str = "", - objective_metrics: list[dict[str, str]] | None = None, - regression_metrics: list[dict[str, str]] | None = None, + objective_metrics: list[dict[str, Any]] | None = None, + regression_metrics: list[dict[str, Any]] | None = None, client: Any = None, workspace: Any = None, ) -> Diagnostic: @@ -127,8 +127,8 @@ async def __call__( agent_id: str, dataset: Any, evaluation: Any, - objective_metrics: list[dict[str, str]], - regression_metrics: list[dict[str, str]], + objective_metrics: list[dict[str, Any]], + regression_metrics: list[dict[str, Any]], ) -> list[TrialSelection]: return [ TrialSelection(trial_id=trial.id, reason=f"Analyze {trial.id} for this test.") for trial in self._trials @@ -141,8 +141,8 @@ async def __call__( agent_id: str, diagnoses: Any, trials: Any, - objective_metrics: list[dict[str, str]], - regression_metrics: list[dict[str, str]], + objective_metrics: list[dict[str, Any]], + regression_metrics: list[dict[str, Any]], ) -> FailureClassification: return FailureClassification(systematic=[], mechanical=[]) @@ -154,8 +154,8 @@ async def __call__( evaluation: Any, diagnoses: Any, peer_evaluations: Any = None, - objective_metrics: list[dict[str, str]] | None = None, - regression_metrics: list[dict[str, str]] | None = None, + objective_metrics: list[dict[str, Any]] | None = None, + regression_metrics: list[dict[str, Any]] | None = None, ) -> PeerComparison: return PeerComparison(divergent_trials=[], complementary_patterns=[]) @@ -204,8 +204,10 @@ def test_peer_comparison_respects_minimize_metric_directions(tmp_path: Path) -> [{"name": "quality", "direction": "maximize"}, {"name": "tokens", "direction": "minimize"}], [] ) - pairs = analyzer._select_divergent_pairs("focal", focal, {"peer": peer}, directions) - complementary = analyzer._find_complementary_failures("focal", focal, {"peer": peer}, directions) + pairs = analyzer._select_divergent_pairs("focal", cast(Any, focal), cast(Any, {"peer": peer}), directions) + complementary = analyzer._find_complementary_failures( + "focal", cast(Any, focal), cast(Any, {"peer": peer}), directions + ) assert pairs[0]["winner"] == "peer" assert complementary["task-1"]["quality"]["leaders"] == ["focal"] @@ -279,6 +281,28 @@ async def test_run_threads_metric_contract_into_trace_analyzer(tmp_path: Path, m assert calls[0]["regression_metrics"] == regression_metrics +@pytest.mark.asyncio +async def test_run_threads_numeric_objective_targets_into_trace_analyzer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Check that trace analysis receives a numeric objective target.""" + calls: list[dict[str, Any]] = [] + _install_fakes(monkeypatch, calls) + trial, dataset, evaluation = _fixtures() + analyzer = _make_analyzer(tmp_path, [trial]) + objective_metrics = [{"name": "reward", "direction": "maximize", "target": 1.0}] + + await analyzer.run( + agent="agent-a", + dataset=cast(Any, dataset), + evaluation=cast(Any, evaluation), + round=0, + objective_metrics=objective_metrics, + ) + + assert calls[0]["objective_metrics"] == objective_metrics + + @pytest.mark.asyncio async def test_intake_availability_is_part_of_cache_key(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """A trace-skipped (no client) result must not be replayed once a client is available."""