Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,26 @@ class MetricTarget(BaseModel):
direction: Literal["maximize", "minimize"] = Field(
description="Whether higher or lower values are better for this target."
)
target: float | None = Field(
default=None,
description=(
"Value at which this objective counts as satisfied, in the metric's own "
"units. When every targeted objective is met the run stops, so a solved "
"problem stops paying for rounds. Unset means no such stop: metrics are not "
"required to be normalized, so there is no value that means 'as good as "
"possible' for an arbitrary one."
),
)

def is_satisfied_by(self, value: float | None) -> bool:
"""Whether *value* meets this target. False when either side is absent.

A missing measurement is not evidence of success, and a target that was never
configured must not end a run.
"""
if value is None or self.target is None:
return False
return value >= self.target if self.direction == "maximize" else value <= self.target


def pareto_objectives(metrics: dict[str, float], objective_function: list[MetricTarget]) -> dict[str, float]:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -272,8 +272,8 @@ async def select_trials(
agent_id: str,
dataset: Dataset,
evaluation: EvaluationResult,
objective_metrics: list[dict[str, str]],
regression_metrics: list[dict[str, str]],
objective_metrics: list[dict[str, Any]],
regression_metrics: list[dict[str, Any]],
) -> list[TrialSelection]:
"""Pick which trials to analyze in depth and explain each choice.

Expand Down Expand Up @@ -351,8 +351,8 @@ async def classify_failures(
agent_id: str,
diagnoses: list[Diagnostic],
trials: Sequence[TrialResult],
objective_metrics: list[dict[str, str]],
regression_metrics: list[dict[str, str]],
objective_metrics: list[dict[str, Any]],
regression_metrics: list[dict[str, Any]],
) -> FailureClassification:
"""Classify diagnoses into systematic vs. one-off and agent vs. mechanical failures.

Expand Down Expand Up @@ -388,8 +388,8 @@ async def compare_with_peers(
evaluation: EvaluationResult,
diagnoses: list[Diagnostic],
peer_evaluations: dict[str, EvaluationResult] | None = None,
objective_metrics: list[dict[str, str]] | None = None,
regression_metrics: list[dict[str, str]] | None = None,
objective_metrics: list[dict[str, Any]] | None = None,
regression_metrics: list[dict[str, Any]] | None = None,
) -> PeerComparison:
"""Compare this agent to peers and return divergent trials and complementary patterns.

Expand Down Expand Up @@ -513,7 +513,7 @@ def _task_metric_means(self, evaluation: EvaluationResult) -> dict[str, dict[str

@staticmethod
def _metric_directions(
objective_metrics: list[dict[str, str]], regression_metrics: list[dict[str, str]]
objective_metrics: list[dict[str, Any]], regression_metrics: list[dict[str, Any]]
) -> dict[str, str]:
"""Return metric directions, defaulting dimensions outside the contract to maximize."""
return {metric["name"]: metric["direction"] for metric in [*objective_metrics, *regression_metrics]}
Expand Down Expand Up @@ -572,8 +572,8 @@ async def _narrate_peer_comparison(
top_divergent: list[dict[str, Any]],
complementary_raw: dict[str, dict[str, dict[str, list[str]]]],
diagnoses: list[Diagnostic],
objective_metrics: list[dict[str, str]],
regression_metrics: list[dict[str, str]],
objective_metrics: list[dict[str, Any]],
regression_metrics: list[dict[str, Any]],
) -> PeerComparison:
"""Write the DivergentTrial and ComplementaryPattern narratives from pre-computed data.

Expand Down Expand Up @@ -652,8 +652,8 @@ async def run(
client: AsyncNeMoPlatform | None = None,
nmp_workspace: str | None = None,
agent_spec: Path | None = None,
objective_metrics: list[dict[str, str]] | None = None,
regression_metrics: list[dict[str, str]] | None = None,
objective_metrics: list[dict[str, Any]] | None = None,
regression_metrics: list[dict[str, Any]] | None = None,
) -> AgentAnalysis:
"""Run the full analysis pipeline for one agent in one optimization round.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,8 @@ async def run(
round_num: int,
phase: Literal["exploration", "exploitation"],
max_candidates: int,
objective_metrics: list[dict[str, str]],
regression_metrics: list[dict[str, str]],
objective_metrics: list[dict[str, Any]],
regression_metrics: list[dict[str, Any]],
) -> list[Improvement]:
"""Return up to max_candidates targeted improvement proposals.

Expand Down Expand Up @@ -242,8 +242,8 @@ async def _run_with_context(
cards_index: str,
phase: Literal["exploration", "exploitation"],
max_candidates: int,
objective_metrics: list[dict[str, str]],
regression_metrics: list[dict[str, str]],
objective_metrics: list[dict[str, Any]],
regression_metrics: list[dict[str, Any]],
) -> list[Improvement]:
"""Pick up to `max_candidates` targeted improvements grounded in root causes.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,9 @@ async def run(
Returns:
A :class:`TerminationDecision`.
"""
reached = self.assess_objective_reached(evolution_tree=evolution_tree, config=config)
if reached.stop:
return reached
budget = self.assess_round_budget(round_num=round_num, config=config)
if budget.stop:
return budget
Expand All @@ -86,6 +89,34 @@ async def run(
config=config,
)

@hidden
def assess_objective_reached(
self,
*,
evolution_tree: EvolutionTree,
config: EvolutionaryOptimizerConfig,
) -> TerminationDecision:
"""Stop when a candidate that could win already satisfies every targeted objective.

Convergence asks whether progress has stalled; this asks whether any is left to
make. Not gated by ``disable_convergence_check``: that disables a judgement about
stagnation, this is a threshold the caller stated. To keep going, set no target.

Consulted before every round, so a baseline that already qualifies costs nothing.
Only survivors and the round-0 baseline count, mirroring finalization -- a killed
candidate would end the run in favour of a winner that never reaches the target.
"""
targets = [target for target in config.objective_function if target.target is not None]
if not targets:
return TerminationDecision(stop=False)
for node in evolution_tree.nodes.values():
if not node.val_reward or not (node.is_survivor or node.round == 0):
continue
if all(target.is_satisfied_by(node.val_reward.get(target.name)) for target in targets):
summary = ", ".join(f"{target.name}={node.val_reward.get(target.name)}" for target in targets)
return TerminationDecision(stop=True, reason=f"objective reached by {node.label} ({summary})")
return TerminationDecision(stop=False)

@hidden
async def assess_convergence(
self,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -194,8 +194,8 @@ async def analyze_trajectory(
runtime: DependencyRuntime | None,
insight: Insight | None = None,
selection_reason: str = "",
objective_metrics: list[dict[str, str]] | None = None,
regression_metrics: list[dict[str, str]] | None = None,
objective_metrics: list[dict[str, Any]] | None = None,
regression_metrics: list[dict[str, Any]] | None = None,
) -> StepAnalysis:
"""Trace the agent's path and find where it went wrong.

Expand Down Expand Up @@ -259,8 +259,8 @@ async def diagnose(
trace: TraceExplorer,
analysis: StepAnalysis,
insight: Insight | None = None,
objective_metrics: list[dict[str, str]] | None = None,
regression_metrics: list[dict[str, str]] | None = None,
objective_metrics: list[dict[str, Any]] | None = None,
regression_metrics: list[dict[str, Any]] | None = None,
) -> Diagnostic:
"""Determine the primary root cause and produce a Diagnostic.

Expand Down Expand Up @@ -302,8 +302,8 @@ async def run(
rationale: Rationale | None = None,
insight: Insight | None = None,
selection_reason: str = "",
objective_metrics: list[dict[str, str]] | None = None,
regression_metrics: list[dict[str, str]] | None = None,
objective_metrics: list[dict[str, Any]] | None = None,
regression_metrics: list[dict[str, Any]] | None = None,
client: AsyncNeMoPlatform | None = None,
workspace: str | None = None,
) -> Diagnostic:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Pin when a solved objective ends a run.

Before this, the only stop conditions were the round budget and a stagnation
judgement. Neither notices success: a round that goes 0.333 -> 1.000 is the
clearest possible case of *not* stagnating, so a solved run kept buying rounds
that could only match what it already had.

The rule: when every objective carrying a ``target`` is satisfied by a candidate
that could win, stop. Absent targets, nothing changes.
"""

from __future__ import annotations

import pytest
from nemo_experimentalist_plugin.config import EvolutionaryOptimizerConfig, MetricTarget
from nemo_experimentalist_plugin.entities import Candidate, RewardRecord
from nemo_experimentalist_plugin.experimentalist.components.models import EvolutionNode, EvolutionTree
from nemo_experimentalist_plugin.experimentalist.components.terminator import Terminator

SOLVED = [MetricTarget(name="reward", direction="maximize", target=1.0)]
UNTARGETED = [MetricTarget(name="reward", direction="maximize")]


def _node(label: str, round_: int, killed: int | None = None, **metrics: float) -> EvolutionNode:
candidate = Candidate(
run_id="run-1",
label=label,
round=round_,
optimization="baseline" if round_ == 0 else "some change",
rewards={"validation": RewardRecord(metrics=dict(metrics))},
)
candidate.killed_round = killed
return EvolutionNode(candidate=candidate)


def _tree(*nodes: EvolutionNode) -> EvolutionTree:
tree = EvolutionTree()
for node in nodes:
tree.add(node.candidate)
return tree


def _assess(tree: EvolutionTree, objectives: list[MetricTarget], **overrides: object):
config = EvolutionaryOptimizerConfig.model_validate(
{"objective_function": [target.model_dump() for target in objectives], **overrides}
)
return Terminator.assess_objective_reached(Terminator, evolution_tree=tree, config=config) # type: ignore[arg-type]


def test_a_solved_objective_stops_the_run() -> None:
decision = _assess(_tree(_node("agent-0", 0, reward=0.333), _node("agent-1", 1, reward=1.0)), SOLVED)
assert decision.stop
assert "agent-1" in decision.reason


def test_an_unsolved_objective_does_not_stop_the_run() -> None:
decision = _assess(_tree(_node("agent-0", 0, reward=0.333), _node("agent-1", 1, reward=0.667)), SOLVED)
assert not decision.stop


def test_no_target_configured_never_stops() -> None:
"""Absent an explicit target there is no value that means 'as good as possible'."""
decision = _assess(_tree(_node("agent-0", 0, reward=1.0), _node("agent-1", 1, reward=1.0)), UNTARGETED)
assert not decision.stop


def test_a_baseline_that_already_meets_the_target_stops_before_any_round() -> None:
"""The terminator is consulted at the top of round 1, so nothing is paid for."""
decision = _assess(_tree(_node("agent-0", 0, reward=1.0)), SOLVED)
assert decision.stop
assert "agent-0" in decision.reason


def test_every_targeted_objective_must_be_met() -> None:
objectives = [
MetricTarget(name="reward", direction="maximize", target=1.0),
MetricTarget(name="shape_ok", direction="maximize", target=1.0),
]
partial = _assess(_tree(_node("agent-1", 1, reward=1.0, shape_ok=0.5)), objectives)
assert not partial.stop
both = _assess(_tree(_node("agent-1", 1, reward=1.0, shape_ok=1.0)), objectives)
assert both.stop


def test_an_untargeted_objective_does_not_block_the_stop() -> None:
"""Only targeted objectives are judged; an untargeted one is not an obstacle."""
objectives = [
MetricTarget(name="reward", direction="maximize", target=1.0),
MetricTarget(name="coverage", direction="maximize"),
]
decision = _assess(_tree(_node("agent-1", 1, reward=1.0, coverage=0.1)), objectives)
assert decision.stop


def test_minimized_objective_uses_the_opposite_comparison() -> None:
objectives = [MetricTarget(name="cost", direction="minimize", target=0.2)]
assert _assess(_tree(_node("agent-1", 1, cost=0.1)), objectives).stop
assert not _assess(_tree(_node("agent-1", 1, cost=0.3)), objectives).stop


def test_a_missing_measurement_is_not_success() -> None:
"""The metric was never reported; absence must not be read as meeting the target."""
decision = _assess(_tree(_node("agent-1", 1, other=1.0)), SOLVED)
assert not decision.stop


def test_a_killed_candidate_cannot_end_the_run() -> None:
"""It cannot win, so stopping on it would ship a winner that never met the target."""
decision = _assess(
_tree(_node("agent-0", 0, reward=0.333), _node("agent-1", 1, killed=1, reward=1.0)),
SOLVED,
)
assert not decision.stop


def test_a_killed_baseline_still_counts() -> None:
"""Finalization re-admits the baseline, so it remains a candidate for winning."""
decision = _assess(_tree(_node("agent-0", 0, killed=1, reward=1.0)), SOLVED)
assert decision.stop


def test_unscored_nodes_are_ignored() -> None:
decision = _assess(_tree(_node("agent-0", 0, reward=0.5), _node("agent-1", 1)), SOLVED)
assert not decision.stop


@pytest.mark.parametrize("disabled", [True, False])
def test_the_stop_is_independent_of_disable_convergence_check(disabled: bool) -> None:
"""That flag turns off a judgement about stagnation; this is a stated threshold."""
decision = _assess(
_tree(_node("agent-1", 1, reward=1.0)),
SOLVED,
disable_convergence_check=disabled,
)
assert decision.stop
Loading
Loading