diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/config.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/config.py index 440c175897..e5dc55da36 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/config.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/config.py @@ -57,6 +57,26 @@ class MetricTarget(BaseModel): direction: Literal["maximize", "minimize"] = Field( description="Whether higher or lower values are better for this target." ) + target: float | None = Field( + default=None, + description=( + "Value at which this objective counts as satisfied, in the metric's own " + "units. When every targeted objective is met the run stops, so a solved " + "problem stops paying for rounds. Unset means no such stop: metrics are not " + "required to be normalized, so there is no value that means 'as good as " + "possible' for an arbitrary one." + ), + ) + + def is_satisfied_by(self, value: float | None) -> bool: + """Whether *value* meets this target. False when either side is absent. + + A missing measurement is not evidence of success, and a target that was never + configured must not end a run. + """ + if value is None or self.target is None: + return False + return value >= self.target if self.direction == "maximize" else value <= self.target def pareto_objectives(metrics: dict[str, float], objective_function: list[MetricTarget]) -> dict[str, float]: diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/analyzer.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/analyzer.py index f8316e179a..566069f758 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/analyzer.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/analyzer.py @@ -272,8 +272,8 @@ async def select_trials( agent_id: str, dataset: Dataset, evaluation: EvaluationResult, - objective_metrics: list[dict[str, str]], - regression_metrics: list[dict[str, str]], + objective_metrics: list[dict[str, Any]], + regression_metrics: list[dict[str, Any]], ) -> list[TrialSelection]: """Pick which trials to analyze in depth and explain each choice. @@ -351,8 +351,8 @@ async def classify_failures( agent_id: str, diagnoses: list[Diagnostic], trials: Sequence[TrialResult], - objective_metrics: list[dict[str, str]], - regression_metrics: list[dict[str, str]], + objective_metrics: list[dict[str, Any]], + regression_metrics: list[dict[str, Any]], ) -> FailureClassification: """Classify diagnoses into systematic vs. one-off and agent vs. mechanical failures. @@ -388,8 +388,8 @@ async def compare_with_peers( evaluation: EvaluationResult, diagnoses: list[Diagnostic], peer_evaluations: dict[str, EvaluationResult] | None = None, - objective_metrics: list[dict[str, str]] | None = None, - regression_metrics: list[dict[str, str]] | None = None, + objective_metrics: list[dict[str, Any]] | None = None, + regression_metrics: list[dict[str, Any]] | None = None, ) -> PeerComparison: """Compare this agent to peers and return divergent trials and complementary patterns. @@ -513,7 +513,7 @@ def _task_metric_means(self, evaluation: EvaluationResult) -> dict[str, dict[str @staticmethod def _metric_directions( - objective_metrics: list[dict[str, str]], regression_metrics: list[dict[str, str]] + objective_metrics: list[dict[str, Any]], regression_metrics: list[dict[str, Any]] ) -> dict[str, str]: """Return metric directions, defaulting dimensions outside the contract to maximize.""" return {metric["name"]: metric["direction"] for metric in [*objective_metrics, *regression_metrics]} @@ -572,8 +572,8 @@ async def _narrate_peer_comparison( top_divergent: list[dict[str, Any]], complementary_raw: dict[str, dict[str, dict[str, list[str]]]], diagnoses: list[Diagnostic], - objective_metrics: list[dict[str, str]], - regression_metrics: list[dict[str, str]], + objective_metrics: list[dict[str, Any]], + regression_metrics: list[dict[str, Any]], ) -> PeerComparison: """Write the DivergentTrial and ComplementaryPattern narratives from pre-computed data. @@ -652,8 +652,8 @@ async def run( client: AsyncNeMoPlatform | None = None, nmp_workspace: str | None = None, agent_spec: Path | None = None, - objective_metrics: list[dict[str, str]] | None = None, - regression_metrics: list[dict[str, str]] | None = None, + objective_metrics: list[dict[str, Any]] | None = None, + regression_metrics: list[dict[str, Any]] | None = None, ) -> AgentAnalysis: """Run the full analysis pipeline for one agent in one optimization round. diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/proposer.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/proposer.py index 6c1d647d5f..331bd28459 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/proposer.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/proposer.py @@ -101,8 +101,8 @@ async def run( round_num: int, phase: Literal["exploration", "exploitation"], max_candidates: int, - objective_metrics: list[dict[str, str]], - regression_metrics: list[dict[str, str]], + objective_metrics: list[dict[str, Any]], + regression_metrics: list[dict[str, Any]], ) -> list[Improvement]: """Return up to max_candidates targeted improvement proposals. @@ -242,8 +242,8 @@ async def _run_with_context( cards_index: str, phase: Literal["exploration", "exploitation"], max_candidates: int, - objective_metrics: list[dict[str, str]], - regression_metrics: list[dict[str, str]], + objective_metrics: list[dict[str, Any]], + regression_metrics: list[dict[str, Any]], ) -> list[Improvement]: """Pick up to `max_candidates` targeted improvements grounded in root causes. diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/terminator.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/terminator.py index 5e12ceb353..46344d68b9 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/terminator.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/terminator.py @@ -77,6 +77,9 @@ async def run( Returns: A :class:`TerminationDecision`. """ + reached = self.assess_objective_reached(evolution_tree=evolution_tree, config=config) + if reached.stop: + return reached budget = self.assess_round_budget(round_num=round_num, config=config) if budget.stop: return budget @@ -86,6 +89,34 @@ async def run( config=config, ) + @hidden + def assess_objective_reached( + self, + *, + evolution_tree: EvolutionTree, + config: EvolutionaryOptimizerConfig, + ) -> TerminationDecision: + """Stop when a candidate that could win already satisfies every targeted objective. + + Convergence asks whether progress has stalled; this asks whether any is left to + make. Not gated by ``disable_convergence_check``: that disables a judgement about + stagnation, this is a threshold the caller stated. To keep going, set no target. + + Consulted before every round, so a baseline that already qualifies costs nothing. + Only survivors and the round-0 baseline count, mirroring finalization -- a killed + candidate would end the run in favour of a winner that never reaches the target. + """ + targets = [target for target in config.objective_function if target.target is not None] + if not targets: + return TerminationDecision(stop=False) + for node in evolution_tree.nodes.values(): + if not node.val_reward or not (node.is_survivor or node.round == 0): + continue + if all(target.is_satisfied_by(node.val_reward.get(target.name)) for target in targets): + summary = ", ".join(f"{target.name}={node.val_reward.get(target.name)}" for target in targets) + return TerminationDecision(stop=True, reason=f"objective reached by {node.label} ({summary})") + return TerminationDecision(stop=False) + @hidden async def assess_convergence( self, diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_analyzer.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_analyzer.py index fc2feb5e73..956eefd057 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_analyzer.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_analyzer.py @@ -194,8 +194,8 @@ async def analyze_trajectory( runtime: DependencyRuntime | None, insight: Insight | None = None, selection_reason: str = "", - objective_metrics: list[dict[str, str]] | None = None, - regression_metrics: list[dict[str, str]] | None = None, + objective_metrics: list[dict[str, Any]] | None = None, + regression_metrics: list[dict[str, Any]] | None = None, ) -> StepAnalysis: """Trace the agent's path and find where it went wrong. @@ -259,8 +259,8 @@ async def diagnose( trace: TraceExplorer, analysis: StepAnalysis, insight: Insight | None = None, - objective_metrics: list[dict[str, str]] | None = None, - regression_metrics: list[dict[str, str]] | None = None, + objective_metrics: list[dict[str, Any]] | None = None, + regression_metrics: list[dict[str, Any]] | None = None, ) -> Diagnostic: """Determine the primary root cause and produce a Diagnostic. @@ -302,8 +302,8 @@ async def run( rationale: Rationale | None = None, insight: Insight | None = None, selection_reason: str = "", - objective_metrics: list[dict[str, str]] | None = None, - regression_metrics: list[dict[str, str]] | None = None, + objective_metrics: list[dict[str, Any]] | None = None, + regression_metrics: list[dict[str, Any]] | None = None, client: AsyncNeMoPlatform | None = None, workspace: str | None = None, ) -> Diagnostic: diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_objective_reached.py b/plugins/nemo-experimentalist/tests/experimentalist/test_objective_reached.py new file mode 100644 index 0000000000..e1ec72a5d1 --- /dev/null +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_objective_reached.py @@ -0,0 +1,138 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pin when a solved objective ends a run. + +Before this, the only stop conditions were the round budget and a stagnation +judgement. Neither notices success: a round that goes 0.333 -> 1.000 is the +clearest possible case of *not* stagnating, so a solved run kept buying rounds +that could only match what it already had. + +The rule: when every objective carrying a ``target`` is satisfied by a candidate +that could win, stop. Absent targets, nothing changes. +""" + +from __future__ import annotations + +import pytest +from nemo_experimentalist_plugin.config import EvolutionaryOptimizerConfig, MetricTarget +from nemo_experimentalist_plugin.entities import Candidate, RewardRecord +from nemo_experimentalist_plugin.experimentalist.components.models import EvolutionNode, EvolutionTree +from nemo_experimentalist_plugin.experimentalist.components.terminator import Terminator + +SOLVED = [MetricTarget(name="reward", direction="maximize", target=1.0)] +UNTARGETED = [MetricTarget(name="reward", direction="maximize")] + + +def _node(label: str, round_: int, killed: int | None = None, **metrics: float) -> EvolutionNode: + candidate = Candidate( + run_id="run-1", + label=label, + round=round_, + optimization="baseline" if round_ == 0 else "some change", + rewards={"validation": RewardRecord(metrics=dict(metrics))}, + ) + candidate.killed_round = killed + return EvolutionNode(candidate=candidate) + + +def _tree(*nodes: EvolutionNode) -> EvolutionTree: + tree = EvolutionTree() + for node in nodes: + tree.add(node.candidate) + return tree + + +def _assess(tree: EvolutionTree, objectives: list[MetricTarget], **overrides: object): + config = EvolutionaryOptimizerConfig.model_validate( + {"objective_function": [target.model_dump() for target in objectives], **overrides} + ) + return Terminator.assess_objective_reached(Terminator, evolution_tree=tree, config=config) # type: ignore[arg-type] + + +def test_a_solved_objective_stops_the_run() -> None: + decision = _assess(_tree(_node("agent-0", 0, reward=0.333), _node("agent-1", 1, reward=1.0)), SOLVED) + assert decision.stop + assert "agent-1" in decision.reason + + +def test_an_unsolved_objective_does_not_stop_the_run() -> None: + decision = _assess(_tree(_node("agent-0", 0, reward=0.333), _node("agent-1", 1, reward=0.667)), SOLVED) + assert not decision.stop + + +def test_no_target_configured_never_stops() -> None: + """Absent an explicit target there is no value that means 'as good as possible'.""" + decision = _assess(_tree(_node("agent-0", 0, reward=1.0), _node("agent-1", 1, reward=1.0)), UNTARGETED) + assert not decision.stop + + +def test_a_baseline_that_already_meets_the_target_stops_before_any_round() -> None: + """The terminator is consulted at the top of round 1, so nothing is paid for.""" + decision = _assess(_tree(_node("agent-0", 0, reward=1.0)), SOLVED) + assert decision.stop + assert "agent-0" in decision.reason + + +def test_every_targeted_objective_must_be_met() -> None: + objectives = [ + MetricTarget(name="reward", direction="maximize", target=1.0), + MetricTarget(name="shape_ok", direction="maximize", target=1.0), + ] + partial = _assess(_tree(_node("agent-1", 1, reward=1.0, shape_ok=0.5)), objectives) + assert not partial.stop + both = _assess(_tree(_node("agent-1", 1, reward=1.0, shape_ok=1.0)), objectives) + assert both.stop + + +def test_an_untargeted_objective_does_not_block_the_stop() -> None: + """Only targeted objectives are judged; an untargeted one is not an obstacle.""" + objectives = [ + MetricTarget(name="reward", direction="maximize", target=1.0), + MetricTarget(name="coverage", direction="maximize"), + ] + decision = _assess(_tree(_node("agent-1", 1, reward=1.0, coverage=0.1)), objectives) + assert decision.stop + + +def test_minimized_objective_uses_the_opposite_comparison() -> None: + objectives = [MetricTarget(name="cost", direction="minimize", target=0.2)] + assert _assess(_tree(_node("agent-1", 1, cost=0.1)), objectives).stop + assert not _assess(_tree(_node("agent-1", 1, cost=0.3)), objectives).stop + + +def test_a_missing_measurement_is_not_success() -> None: + """The metric was never reported; absence must not be read as meeting the target.""" + decision = _assess(_tree(_node("agent-1", 1, other=1.0)), SOLVED) + assert not decision.stop + + +def test_a_killed_candidate_cannot_end_the_run() -> None: + """It cannot win, so stopping on it would ship a winner that never met the target.""" + decision = _assess( + _tree(_node("agent-0", 0, reward=0.333), _node("agent-1", 1, killed=1, reward=1.0)), + SOLVED, + ) + assert not decision.stop + + +def test_a_killed_baseline_still_counts() -> None: + """Finalization re-admits the baseline, so it remains a candidate for winning.""" + decision = _assess(_tree(_node("agent-0", 0, killed=1, reward=1.0)), SOLVED) + assert decision.stop + + +def test_unscored_nodes_are_ignored() -> None: + decision = _assess(_tree(_node("agent-0", 0, reward=0.5), _node("agent-1", 1)), SOLVED) + assert not decision.stop + + +@pytest.mark.parametrize("disabled", [True, False]) +def test_the_stop_is_independent_of_disable_convergence_check(disabled: bool) -> None: + """That flag turns off a judgement about stagnation; this is a stated threshold.""" + decision = _assess( + _tree(_node("agent-1", 1, reward=1.0)), + SOLVED, + disable_convergence_check=disabled, + ) + assert decision.stop diff --git a/plugins/nemo-experimentalist/tests/test_experimentalist_analyzer.py b/plugins/nemo-experimentalist/tests/test_experimentalist_analyzer.py index 7bac381067..7f766e5599 100644 --- a/plugins/nemo-experimentalist/tests/test_experimentalist_analyzer.py +++ b/plugins/nemo-experimentalist/tests/test_experimentalist_analyzer.py @@ -89,8 +89,8 @@ async def run( rationale: Any = None, insight: Any = None, selection_reason: str = "", - objective_metrics: list[dict[str, str]] | None = None, - regression_metrics: list[dict[str, str]] | None = None, + objective_metrics: list[dict[str, Any]] | None = None, + regression_metrics: list[dict[str, Any]] | None = None, client: Any = None, workspace: Any = None, ) -> Diagnostic: @@ -127,8 +127,8 @@ async def __call__( agent_id: str, dataset: Any, evaluation: Any, - objective_metrics: list[dict[str, str]], - regression_metrics: list[dict[str, str]], + objective_metrics: list[dict[str, Any]], + regression_metrics: list[dict[str, Any]], ) -> list[TrialSelection]: return [ TrialSelection(trial_id=trial.id, reason=f"Analyze {trial.id} for this test.") for trial in self._trials @@ -141,8 +141,8 @@ async def __call__( agent_id: str, diagnoses: Any, trials: Any, - objective_metrics: list[dict[str, str]], - regression_metrics: list[dict[str, str]], + objective_metrics: list[dict[str, Any]], + regression_metrics: list[dict[str, Any]], ) -> FailureClassification: return FailureClassification(systematic=[], mechanical=[]) @@ -154,8 +154,8 @@ async def __call__( evaluation: Any, diagnoses: Any, peer_evaluations: Any = None, - objective_metrics: list[dict[str, str]] | None = None, - regression_metrics: list[dict[str, str]] | None = None, + objective_metrics: list[dict[str, Any]] | None = None, + regression_metrics: list[dict[str, Any]] | None = None, ) -> PeerComparison: return PeerComparison(divergent_trials=[], complementary_patterns=[]) @@ -204,8 +204,10 @@ def test_peer_comparison_respects_minimize_metric_directions(tmp_path: Path) -> [{"name": "quality", "direction": "maximize"}, {"name": "tokens", "direction": "minimize"}], [] ) - pairs = analyzer._select_divergent_pairs("focal", focal, {"peer": peer}, directions) - complementary = analyzer._find_complementary_failures("focal", focal, {"peer": peer}, directions) + pairs = analyzer._select_divergent_pairs("focal", cast(Any, focal), cast(Any, {"peer": peer}), directions) + complementary = analyzer._find_complementary_failures( + "focal", cast(Any, focal), cast(Any, {"peer": peer}), directions + ) assert pairs[0]["winner"] == "peer" assert complementary["task-1"]["quality"]["leaders"] == ["focal"] @@ -279,6 +281,28 @@ async def test_run_threads_metric_contract_into_trace_analyzer(tmp_path: Path, m assert calls[0]["regression_metrics"] == regression_metrics +@pytest.mark.asyncio +async def test_run_threads_numeric_objective_targets_into_trace_analyzer( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Check that trace analysis receives a numeric objective target.""" + calls: list[dict[str, Any]] = [] + _install_fakes(monkeypatch, calls) + trial, dataset, evaluation = _fixtures() + analyzer = _make_analyzer(tmp_path, [trial]) + objective_metrics = [{"name": "reward", "direction": "maximize", "target": 1.0}] + + await analyzer.run( + agent="agent-a", + dataset=cast(Any, dataset), + evaluation=cast(Any, evaluation), + round=0, + objective_metrics=objective_metrics, + ) + + assert calls[0]["objective_metrics"] == objective_metrics + + @pytest.mark.asyncio async def test_intake_availability_is_part_of_cache_key(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """A trace-skipped (no client) result must not be replayed once a client is available.""" diff --git a/plugins/nemo-experimentalist/tests/test_metric_contract.py b/plugins/nemo-experimentalist/tests/test_metric_contract.py index bb01578c81..753967a869 100644 --- a/plugins/nemo-experimentalist/tests/test_metric_contract.py +++ b/plugins/nemo-experimentalist/tests/test_metric_contract.py @@ -97,7 +97,11 @@ def test_insight_metrics_become_objectives_and_existing_targets_become_guardrail assert [target.name for target in effective.objective_function] == ["uses_required_tool", "cites_source"] assert all(target.direction == "maximize" for target in effective.objective_function) + # Demoted targets are carried across whole, `target` included. The authored insight + # objectives get none: an LLM-invented metric has no known satisfied value, so a + # Mode 1 run cannot stop early on one and falls back to its round budget. assert [target.model_dump() for target in effective.regression_metrics] == [ - {"name": "cost", "direction": "minimize"}, - {"name": "safety", "direction": "maximize"}, + {"name": "cost", "direction": "minimize", "target": None}, + {"name": "safety", "direction": "maximize", "target": None}, ] + assert all(target.target is None for target in effective.objective_function)