diff --git a/plugins/nemo-eval-author/README.md b/plugins/nemo-eval-author/README.md index 850caf4984..17819e18aa 100644 --- a/plugins/nemo-eval-author/README.md +++ b/plugins/nemo-eval-author/README.md @@ -33,7 +33,7 @@ from nemo_eval_author_plugin.eval_author.run import run_eval_author # Still borrowed from Experimentalist, and on the way out. Treat these as Eval Author's # own types once they move; do not build new code on the Experimentalist paths. from nemo_experimentalist_plugin.entities import Dataset, DatasetRef -from nemo_experimentalist_plugin.experimentalist.components.dataset_staging import stage_task_template +from nemo_experimentalist_plugin.experimentalist.components.dataset_staging import stage_eval_author_inputs from nemo_experimentalist_plugin.experimentalist.components.trace_analyzer import TraceAnalyzer from nemo_experimentalist_plugin.experimentalist.components.trace_explorer import TraceExplorer ``` diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/README.md b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/README.md index 15b406c9fa..35c869f87d 100644 --- a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/README.md +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/README.md @@ -55,18 +55,16 @@ Each Eval Author invocation fills a fresh candidate suite from the current templ and traces. The complete suite is Harbor-validated locally, promoted to the experiment-local working copy with backup-and-restore failure handling, and analyzed for the Insight's root cause. Eval Author then adds normalized -Insight-specific verifier metric keys to every materialized task while -preserving the template's existing task metrics. The metric authoring step is -scoped to the Insight suite; the user's train and validation datasets remain -unchanged. The authored verifiers must pass static Harbor validation before the -local suite is returned to the optimization loop. +Insight-specific verifier metric keys to every task in the staged train, +validation, and generated Insight datasets while preserving existing task +metrics. All three datasets must pass static Harbor validation before they are +returned. After authoring and validation, Eval Author hashes every task file and verifier file and persists deterministic suite and scorer identities in the local suite's -manifest. The returned dataset continues to point at the single experiment-local -suite. Candidate Insight results persist the suite identity, and resume reuses -those results only when the identity still matches; changed task or verifier -content is re-evaluated. +manifest. The returned Insight dataset points at the experiment-local suite. +Eval Author does not split, merge, or evaluate that suite. Experimentalist +currently leaves it unconsumed for downstream integration by its owning team. Task-template inputs may be local paths, `file://` URIs, or NeMo Platform `fileset:///` references. Fileset-backed templates are @@ -76,12 +74,6 @@ them. The staged template is refreshed on every invocation rather than reused. The returned Python contract is documented in the [Eval Author Python Reference](REFERENCE.md#evalauthorresult). -Insight metrics remain adaptive development feedback. They may steer round -analysis, goal-tree updates, and proposals, but validation remains the direct -Pareto and winner-selection criterion. Promotion suggestions require complete -repeated baseline-to-winner improvement evidence, remain advisory, and never -mutate the canonical validation dataset. - ## Intended Invocation Until a CLI or platform job is wired, Python callers can invoke the runner diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/REFERENCE.md b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/REFERENCE.md index aa3c584557..96da225a38 100644 --- a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/REFERENCE.md +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/REFERENCE.md @@ -11,13 +11,13 @@ SPDX-License-Identifier: Apache-2.0 | Field | Type | Description | | --- | --- | --- | -| `train_dataset` | `Dataset` | Training dataset supplied to the run. Eval Author does not mutate it. | -| `validation_dataset` | `Dataset` | Validation dataset supplied to the run. Eval Author does not mutate it. | -| `insight_suite` | `Dataset \| None` | Finalized experiment-local Insight dataset for immediate evaluation by the optimization loop. | +| `train_dataset` | `Dataset` | Staged training dataset with the authored metrics and verifier changes. | +| `validation_dataset` | `Dataset` | Staged validation dataset with the authored metrics and verifier changes. | +| `insight_suite` | `Dataset \| None` | Finalized experiment-local task set generated from the Insight's trace references. | | `insight_suite_identity` | `str \| None` | SHA-256 identity of the finalized Insight task and verifier content. | +| `metric_keys` | `tuple[str, ...]` | Metric keys authored across all three datasets. | | `summary` | `str` | Eval Author's analysis summary. | When an Insight suite is materialized successfully, `insight_suite` and -`insight_suite_identity` are both populated. Callers can persist the identity -with candidate results and reuse those results only while the suite identity -continues to match. +`insight_suite_identity` are both populated. Eval Author does not split, merge, +or evaluate the returned suite; its caller owns that downstream integration. diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/agent.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/agent.py index 967dff6f44..0144a61c54 100644 --- a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/agent.py +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/agent.py @@ -12,14 +12,20 @@ from pathlib import Path from typing import Any -from nemo_eval_author_plugin.eval_author.materialization import InsightSuite -from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig, EvalAuthorResult +from nemo_eval_author_plugin.eval_author.materialization import InsightSuite, validate_metric_contracts +from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig, EvalAuthorResult, MetricAuthoringResult from nemo_eval_author_plugin.model_config import ( bridge_author_env_to_experimentalist, get_fast_model, get_smart_model, ) -from nemo_experimentalist_plugin.entities import Dataset, DatasetValidationError, ResourceRef, Task, TrialResult +from nemo_experimentalist_plugin.entities import ( + Dataset, + DatasetValidationError, + ResourceRef, + Task, + TrialResult, +) from nemo_experimentalist_plugin.experimentalist.components import cache from nemo_experimentalist_plugin.experimentalist.components.tools import GuardedShellTools from nemo_experimentalist_plugin.experimentalist.components.trace_analyzer import ( @@ -115,15 +121,19 @@ async def author_insight_metrics( self, insight: Insight, diagnostics: list[tuple[str, Diagnostic]], + train_dataset: Dataset, + validation_dataset: Dataset, insight_suite: Dataset, runner_conventions: str, validation_feedback: str | None = None, - ) -> str: + ) -> MetricAuthoringResult: """Author verifier metrics for the materialized tasks that capture the insight. Args: insight: The insight whose failure mode the tasks should detect. diagnostics: Per-trace ``(trace_ref, Diagnostic)`` pairs for concrete evidence. + train_dataset: Staged training tasks to augment for optimization feedback. + validation_dataset: Staged validation tasks to augment for scoring. insight_suite: The materialized tasks recreated from the Insight's production traces. runner_conventions: Summary of how this dataset's runner works (from ``discover_runner``). Use this as the authoritative reference for what artifacts exist at @@ -135,33 +145,34 @@ async def author_insight_metrics( Refer to ``self.context["dataset_documentation"]`` for the dataset-specific API and metric authoring conventions (file layout, how to add/remove/modify a metric). - **Scope: new grades on the materialized Insight tasks** + **Scope: every task in all three datasets** - Add at least one new Insight-specific metric key to every task in ``insight_suite``. - Use the same new metric key set and shared scoring semantics across the entire - suite. Preserve every existing verifier metric, including the task's ordinary - ``reward`` or ``score``; append the Insight signal instead of replacing the - task's original notion of success. + Add at least one new Insight-specific metric key to every task in + ``train_dataset``, ``validation_dataset``, and ``insight_suite``. Use the same + new metric key set and scoring semantics everywhere. Preserve every existing + verifier metric, including ordinary ``reward`` or ``score`` values. - Only edit verifier files in the materialized Insight suite. Do not modify the - user's train or validation datasets, and do not change task instructions, - environments, solutions, or other agent-visible inputs. This work adds new - grades to the new rows; it does not add new agent output to old benchmark rows. + Task-specific verifier edits are allowed and expected when existing verifier + layouts differ. Only edit verifier files. Do not modify task instructions, + environments, solutions, or other agent-visible inputs. Name each new metric after the root-cause behavior, not a trace id or surface symptom. Measure the current Harbor run from runtime artifacts such as OTLP traces or agent outputs. Do not hard-code scores for the production traces that motivated the Insight. + In every task's configured verifier directory, write ``metric-contract.json`` + containing exactly ``{"metric_keys": ["key_one", "key_two"]}``. The list must + contain the same newly authored keys for every task and must exactly match the + ``metric_keys`` returned in ``MetricAuthoringResult``. + **Validate while authoring** - After every verifier edit, call ``await insight_suite.validate()``. This performs - evaluator-specific static checks without launching trials or executing verifier - code. If it raises ``DatasetValidationError``, use its task, path, and source - location diagnostics to repair the files, then call it again. Do not return until - the suite passes validation. If ``validation_feedback`` is provided, the caller's - mandatory validation found errors in the previous attempt; fix every reported - failure and revalidate the suite. + After verifier edits, call ``await train_dataset.validate()``, + ``await validation_dataset.validate()``, and ``await insight_suite.validate()``. + These perform evaluator-specific static checks without launching trials. If one + raises ``DatasetValidationError``, repair its diagnostics and revalidate all three. + If ``validation_feedback`` is provided, fix every reported failure before returning. **Metric quality** @@ -187,9 +198,9 @@ async def author_insight_metrics( objects for Y, so X is missing from its context. Measure whether the agent retrieves all required objects, not merely whether X appears in the final answer. - Return a concise summary naming the new metric key(s), what they measure, and - which runtime evidence they score. The caller retains the materialized suite and - the user's unchanged train and validation datasets. + Return ``MetricAuthoringResult`` with the unique, non-empty ``metric_keys`` added + by the verifier edits plus a concise summary. Include at least one + Insight-specific key beyond generic ``reward`` or ``score``. """ # noqa: D413 ... @@ -282,8 +293,8 @@ async def _run( insight: The Insight to investigate with relevant traces. agent_path: Agent root, relative to ``experiment_dir`` or absolute. task_template: Parsed evaluator task containing explicit placeholders. - train_dataset: The train dataset, returned unchanged. - validation_dataset: The validation dataset, returned unchanged. + train_dataset: Staged training dataset to augment in place. + validation_dataset: Staged validation dataset to augment in place. client: Existing NeMo Platform client used for Intake requests. """ reporter = self._reporter @@ -391,18 +402,42 @@ async def _run( self.context["dataset_documentation"] = doc(type(materialized_dataset), inline_depth=1) if reporter is not None: reporter.progress(phase="eval author · discovering runner") - runner_conventions = await self.discover_runner(materialized_dataset) + runner_conventions = await self.discover_runner(train_dataset) if reporter is not None: reporter.progress(phase="eval author · authoring metrics") - summary = await self.author_insight_metrics( - insight, - diagnostics, - materialized_dataset, - runner_conventions, - ) + + validation_feedback: str | None = None for repair_attempt in range(self._config.max_validation_repair_attempts + 1): + authoring_result = await self.author_insight_metrics( + insight, + diagnostics, + train_dataset, + validation_dataset, + materialized_dataset, + runner_conventions, + validation_feedback=validation_feedback, + ) try: - await materialized_dataset.validate() + validation_errors: list[str] = [] + for label, dataset in ( + ("train", train_dataset), + ("validation", validation_dataset), + ("insight", materialized_dataset), + ): + try: + await dataset.validate() + except DatasetValidationError as exc: + validation_errors.append(f"{label} dataset:\n{exc}") + if validation_errors: + raise DatasetValidationError("\n".join(validation_errors)) + validate_metric_contracts( + { + "train": train_dataset, + "validation": validation_dataset, + "insight": materialized_dataset, + }, + metric_keys=authoring_result.metric_keys, + ) except DatasetValidationError as exc: if repair_attempt >= self._config.max_validation_repair_attempts: raise @@ -419,24 +454,20 @@ async def _run( total=self._config.max_validation_repair_attempts, unit="attempt", ) - summary = await self.author_insight_metrics( - insight, - diagnostics, - materialized_dataset, - runner_conventions, - validation_feedback=str(exc), - ) - else: - finalized_suite = insight_suite.finalize() - if reporter is not None: - reporter.progress(phase="eval author · complete") - return EvalAuthorResult( - train_dataset=train_dataset, - validation_dataset=validation_dataset, - insight_suite=finalized_suite.dataset, - insight_suite_identity=finalized_suite.identity, - summary=summary, - ) + validation_feedback = str(exc) + continue + + finalized_suite = insight_suite.finalize() + if reporter is not None: + reporter.progress(phase="eval author · complete") + return EvalAuthorResult( + train_dataset=train_dataset, + validation_dataset=validation_dataset, + insight_suite=finalized_suite.dataset, + insight_suite_identity=finalized_suite.identity, + metric_keys=authoring_result.metric_keys, + summary=authoring_result.summary, + ) raise AssertionError("unreachable") diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/materialization.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/materialization.py index 60e2b475ad..eba34453a6 100644 --- a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/materialization.py +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/materialization.py @@ -18,12 +18,13 @@ import tomlkit from harbor.models.task.task import Task as HarborTask -from nemo_experimentalist_plugin.entities import Task, local_path_from_uri +from nemo_experimentalist_plugin.entities import Dataset, DatasetValidationError, Task, local_path_from_uri from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import HarborDataset _MANIFEST_SCHEMA_VERSION = 3 _CONTENT_HASH_SCHEMA_VERSION = 1 _METRIC_CONTRACT_VERSION = 1 +_METRIC_CONTRACT_FILENAME = "metric-contract.json" _SLUG_RE = re.compile(r"[^a-z0-9]+") @@ -67,6 +68,61 @@ def _verifier_dir(task_dir: Path) -> Path: raise ValueError(f"Materialized task has no verifier directory: {task_dir}") +def validate_metric_contracts( + datasets: dict[str, Dataset], + *, + metric_keys: tuple[str, ...], +) -> None: + """Require every task to declare the same newly authored metric keys.""" + expected = set(metric_keys) + failures: list[str] = [] + for dataset_name, dataset in datasets.items(): + for task in dataset.list_tasks(): + if not task.uri: + failures.append(f"{dataset_name}/{task.id}: task URI is missing") + continue + try: + task_dir = local_path_from_uri( + task.uri, + context=f"{dataset_name} dataset task {task.id!r}", + ).resolve() + verifier_dir = _verifier_dir(task_dir) + except (FileNotFoundError, ValueError, tomllib.TOMLDecodeError) as exc: + failures.append(f"{dataset_name}/{task.id}: {exc}") + continue + + contract_path = verifier_dir / _METRIC_CONTRACT_FILENAME + if not contract_path.is_file(): + failures.append(f"{dataset_name}/{task.id}: missing {contract_path}") + continue + try: + payload = json.loads(contract_path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as exc: + failures.append(f"{dataset_name}/{task.id}: invalid JSON in {contract_path}: {exc}") + continue + if not isinstance(payload, dict) or set(payload) != {"metric_keys"}: + failures.append(f"{dataset_name}/{task.id}: {contract_path} must contain only a metric_keys list") + continue + raw_keys = payload["metric_keys"] + if ( + not isinstance(raw_keys, list) + or any(not isinstance(key, str) or not key.strip() for key in raw_keys) + or len(raw_keys) != len(set(raw_keys)) + ): + failures.append( + f"{dataset_name}/{task.id}: {contract_path} metric_keys must be unique non-empty strings" + ) + continue + actual = set(raw_keys) + if actual != expected: + failures.append( + f"{dataset_name}/{task.id}: {contract_path} declares {sorted(actual)}, expected {sorted(expected)}" + ) + if failures: + details = "\n".join(f"- {failure}" for failure in failures) + raise DatasetValidationError(f"Authored metric contract validation failed:\n{details}") + + def _content_provenance(suite_dir: Path, manifest: dict[str, object]) -> tuple[list[dict[str, object]], str, str]: raw_tasks = manifest.get("tasks") if not isinstance(raw_tasks, list): diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/models.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/models.py index 6218d4f708..67dba19207 100644 --- a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/models.py +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/models.py @@ -1,10 +1,51 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Shared models for the top-level Eval Author.""" +"""Small boundary models for Eval Author.""" + +from typing import Self from nemo_experimentalist_plugin.entities import Dataset -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +_GENERIC_METRIC_KEYS = frozenset({"reward", "score"}) + + +def _validate_metric_keys(value: tuple[str, ...]) -> tuple[str, ...]: + keys = tuple(key.strip() for key in value) + if not keys or any(not key for key in keys): + raise ValueError("metric keys must be non-empty") + if len(set(keys)) != len(keys): + raise ValueError("metric keys must be unique") + if set(keys) <= _GENERIC_METRIC_KEYS: + raise ValueError("at least one non-generic metric key is required") + return keys + + +def _non_empty(value: str, *, label: str) -> str: + value = value.strip() + if not value: + raise ValueError(f"{label} must be non-empty") + return value + + +class MetricAuthoringResult(BaseModel): + """Metric keys and a short account of the authored verifier edits.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + metric_keys: tuple[str, ...] + summary: str + + @field_validator("metric_keys") + @classmethod + def _metric_keys_are_portable(cls, value: tuple[str, ...]) -> tuple[str, ...]: + return _validate_metric_keys(value) + + @field_validator("summary") + @classmethod + def _summary_is_non_empty(cls, value: str) -> str: + return _non_empty(value, label="metric authoring summary") class EvalAuthorConfig(BaseModel): @@ -27,18 +68,38 @@ class EvalAuthorConfig(BaseModel): class EvalAuthorResult(BaseModel): - """Output of one Eval Author run.""" + """Modified evaluation datasets, their metric keys, and a summary.""" - model_config = ConfigDict(arbitrary_types_allowed=True) + model_config = ConfigDict(frozen=True, extra="forbid", arbitrary_types_allowed=True) train_dataset: Dataset validation_dataset: Dataset - insight_suite: Dataset | None = Field( - default=None, - description="Finalized experiment-local Insight dataset for use by the optimization loop.", - ) - insight_suite_identity: str | None = Field( - default=None, - description="SHA-256 identity of the finalized Insight task and verifier content.", - ) + insight_suite: Dataset | None = None + insight_suite_identity: str | None = None + metric_keys: tuple[str, ...] = () summary: str + + @field_validator("metric_keys") + @classmethod + def _metric_keys_are_portable(cls, value: tuple[str, ...]) -> tuple[str, ...]: + if not value: + return () + return _validate_metric_keys(value) + + @field_validator("summary") + @classmethod + def _summary_is_non_empty(cls, value: str) -> str: + return _non_empty(value, label="Eval Author result summary") + + @model_validator(mode="after") + def _authored_suite_fields_are_all_or_none(self) -> Self: + if self.insight_suite is None: + if self.insight_suite_identity is not None or self.metric_keys: + raise ValueError("Insight suite identity and metric keys require an authored Insight suite") + return self + if self.insight_suite_identity is None: + raise ValueError("an authored Insight suite requires its content identity") + _non_empty(self.insight_suite_identity, label="Insight suite identity") + if not self.metric_keys: + raise ValueError("an authored Insight suite requires declared metric keys") + return self diff --git a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/run.py b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/run.py index 7a8f918e73..179bd45870 100644 --- a/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/run.py +++ b/plugins/nemo-eval-author/src/nemo_eval_author_plugin/eval_author/run.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Reusable optimizer Eval Author run orchestration.""" +"""Reusable Eval Author run orchestration.""" import importlib from pathlib import Path @@ -10,7 +10,7 @@ from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig, EvalAuthorResult from nemo_experimentalist_plugin.client import make_client from nemo_experimentalist_plugin.entities import Dataset, DatasetRef, Task -from nemo_experimentalist_plugin.experimentalist.components.dataset_staging import stage_task_template +from nemo_experimentalist_plugin.experimentalist.components.dataset_staging import stage_eval_author_inputs from nemo_experimentalist_plugin.experimentalist.components.evaluator.base import EvaluatorType from nemo_experimentalist_plugin.experimentalist.components.evaluator.factory import DatasetFactory from nemo_experimentalist_plugin.experimentalist.experimentalist_backend import ( @@ -51,28 +51,28 @@ async def run_eval_author( agent: Path | str | None = None, evaluator_type: EvaluatorType = "harbor", ) -> EvalAuthorResult: - """Build and run the Eval Author against an Insight and evaluator datasets. + """Stage evaluation inputs, resolve one Insight, then run Eval Author. Args: - insight: Local Insight file path or platform insight id. - train_dataset: Evaluator dataset reference for training. - validation_dataset: Evaluator dataset reference for validation. - task_template: Local or Fileset-backed evaluator task template used for production traces. - experiment_dir: Working directory for Eval Author artifacts. + insight: Local Insight path or platform Insight id. + train_dataset: Training dataset to stage and augment. + validation_dataset: Validation dataset to stage and augment. + task_template: Local or Fileset-backed evaluator task template. + experiment_dir: Working directory for authored artifacts. workspace: Platform workspace. base_url: Platform base URL. ``None`` uses the active platform context. config: Eval Author tuning parameters. - agent: Optional agent source override. When absent, the Insight's agent is used. - evaluator_type: Evaluator adapter used to parse datasets and task template. + agent: Optional agent source override. The Insight's agent is the default. + evaluator_type: Evaluator adapter used to parse the task template. Returns: - Typed Eval Author output containing the train dataset, validation dataset, and summary. + EvalAuthorResult: containing the modified and newly created datasets, additional metrics + and summary. """ _enable_litellm_drop_params() - experiment_dir.mkdir(parents=True, exist_ok=True) experiment_dir = experiment_dir.resolve() - insight = insight.resolve() if isinstance(insight, Path) else insight + insight_locator = str(insight.resolve()) if isinstance(insight, Path) else insight client = make_client(base_url) try: @@ -80,18 +80,39 @@ async def run_eval_author( client=client, experiments_output=str(experiment_dir), ) - resolved_insight = await backend.get_insight(workspace=workspace, insight_id=str(insight)) + resolved_insight = await backend.get_insight( + workspace=workspace, + insight_id=insight_locator, + ) agent_ref = agent if agent is not None else resolved_insight.agent agent_path = experiment_dir / "eval_author" / "source-agent" - await backend.get_agent_code(workspace=workspace, agent=agent_ref, dest=agent_path) + await backend.get_agent_code( + workspace=workspace, + agent=agent_ref, + dest=agent_path, + ) - dataset_factory = DatasetFactory() - staged_task_template = await stage_task_template( + staged_inputs = await stage_eval_author_inputs( experiment_dir, - task_template, + train_dataset=train_dataset, + validation_dataset=validation_dataset, + task_template=task_template, client=client, workspace=workspace, ) + dataset_factory = DatasetFactory() + parsed_train = dataset_factory.build_dataset( + evaluator_type, + staged_inputs.train_dataset, + ) + parsed_validation = dataset_factory.build_dataset( + evaluator_type, + staged_inputs.validation_dataset, + ) + parsed_template = dataset_factory.build_task_template( + evaluator_type, + staged_inputs.task_template, + ) eval_author = build_eval_author_agent( experiment_dir=experiment_dir, config=config, @@ -99,9 +120,9 @@ async def run_eval_author( return await eval_author.run( insight=resolved_insight, agent_path=agent_path, - task_template=dataset_factory.build_task_template(evaluator_type, staged_task_template), - train_dataset=dataset_factory.build_dataset(evaluator_type, train_dataset), - validation_dataset=dataset_factory.build_dataset(evaluator_type, validation_dataset), + task_template=parsed_template, + train_dataset=parsed_train, + validation_dataset=parsed_validation, client=client, ) finally: @@ -117,7 +138,11 @@ def build_eval_author_agent( """Build the LLM-backed Eval Author agent lazily.""" from nemo_eval_author_plugin.eval_author.agent import build_eval_author_agent as _build_eval_author_agent - return _build_eval_author_agent(experiment_dir=experiment_dir, config=config, reporter=reporter) + return _build_eval_author_agent( + experiment_dir=experiment_dir, + config=config, + reporter=reporter, + ) def _enable_litellm_drop_params() -> None: diff --git a/plugins/nemo-eval-author/tests/test_eval_author_agent.py b/plugins/nemo-eval-author/tests/test_eval_author_agent.py index 7eb3b0c076..6fbff706e7 100644 --- a/plugins/nemo-eval-author/tests/test_eval_author_agent.py +++ b/plugins/nemo-eval-author/tests/test_eval_author_agent.py @@ -1,11 +1,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Deterministic contract tests for the canonical top-level Eval Author agent.""" +"""End-to-end control flow for the Eval Author agent.""" import asyncio import inspect -import io from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path @@ -15,8 +14,8 @@ import pytest from nemo_eval_author_plugin.eval_author import agent as eval_author_module from nemo_eval_author_plugin.eval_author.agent import EvalAuthor -from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig -from nemo_experimentalist_plugin.entities import Dataset, DatasetValidationError, Task, TrialResult +from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig, MetricAuthoringResult +from nemo_experimentalist_plugin.entities import Dataset, DatasetValidationError, ResourceRef, Task, TrialResult from nemo_experimentalist_plugin.experimentalist.components.trace_analyzer import ( Diagnostic, TraceAnalyzerConfig, @@ -24,41 +23,6 @@ from nemo_insights_plugin.entities import Insight -@dataclass -class _FillTaskTemplateCall: - trace_ref: str - task_template: Task - client: Any - workspace: str - result: Task - - -@dataclass -class _AnalyzerInitCall: - experiment_dir: Path - config: TraceAnalyzerConfig - - -@dataclass -class _AnalyzerRunCall: - trial: TrialResult - task: Task - agent_path: Path - insight: Insight - client: Any - workspace: str - - -@dataclass -class _PipelineCalls: - fill_task_template: list[_FillTaskTemplateCall] - analyzer_init: list[_AnalyzerInitCall] - analyzer_run: list[_AnalyzerRunCall] - discovered_datasets: list[Dataset] - author_args: list[tuple[Insight, list[tuple[str, Diagnostic]], Dataset, str, str | None]] - suite_discards: int - - @dataclass class _ClosingShell: close_calls: int = 0 @@ -67,6 +31,39 @@ async def close(self) -> None: self.close_calls += 1 +@dataclass +class _Calls: + events: list[str] + filled_refs: list[str] + analyzed_refs: list[str] + diagnostics: list[tuple[str, Diagnostic]] + author_feedback: list[str | None] + train_dataset: Dataset + validation_dataset: Dataset + insight_suite: Dataset + suite_discards: int = 0 + + +class _MaterializedDataset(Dataset): + def __init__( + self, + dataset_id: str, + root: Path, + events: list[str], + validation_errors: list[str] | None = None, + ) -> None: + root.mkdir(parents=True) + (root / "dataset.txt").write_text(dataset_id, encoding="utf-8") + super().__init__(id=dataset_id, source=ResourceRef(uri=root.as_uri())) + self.events = events + self.validation_errors = validation_errors or [] + + async def validate(self) -> None: + self.events.append(f"validate:{self.id}") + if self.validation_errors: + raise DatasetValidationError(self.validation_errors.pop(0)) + + def _insight(trace_refs: list[str], *, insight_id: str = "insight-1") -> Insight: insight = Insight( workspace="workspace-a", @@ -79,173 +76,108 @@ def _insight(trace_refs: list[str], *, insight_id: str = "insight-1") -> Insight return insight +def _diagnostic(summary: str = "wrong tool") -> Diagnostic: + return Diagnostic(outcome="FAILURE", summary=summary, failure_point=1, root_cause="wrong tool") + + def _eval_author( tmp_path: Path, *, max_traces: int = 10, - max_summary_tokens: int = 80_000, max_validation_repair_attempts: int = 5, - reporter: Any = None, ) -> EvalAuthor: eval_author = object.__new__(EvalAuthor) eval_author.experiment_dir = tmp_path eval_author._config = EvalAuthorConfig( max_traces=max_traces, - max_summary_tokens=max_summary_tokens, max_validation_repair_attempts=max_validation_repair_attempts, ) - eval_author._reporter = reporter + eval_author._reporter = None eval_author.context = {} eval_author.shell = cast(Any, _ClosingShell()) return eval_author -def _diagnostic(summary: str) -> Diagnostic: - return Diagnostic(outcome="FAILURE", summary=summary, failure_point=1, root_cause="wrong tool") - - -def _prompt(method: Any) -> str: - prompt = inspect.getdoc(method) - assert prompt is not None - return " ".join(prompt.split()) - - -def test_eval_author_prompts_scope_metrics_to_materialized_insight_suite() -> None: - discover_prompt = _prompt(EvalAuthor.discover_runner) - author_prompt = _prompt(EvalAuthor.author_insight_metrics) - - assert "read it first" in discover_prompt - assert "inspect the actual files" in discover_prompt - assert "authoritative reference for what artifacts exist at evaluation runtime" in author_prompt - assert "how tasks are structured, and how to add metrics" in author_prompt - assert "Add at least one new Insight-specific metric key to every task in ``insight_suite``" in author_prompt - assert "Preserve every existing verifier metric" in author_prompt - assert "Do not modify the user's train or validation datasets" in author_prompt - assert "call ``await insight_suite.validate()``" in author_prompt - assert "fix every reported failure and revalidate the suite" in author_prompt - - -def test_eval_author_prompts_retain_root_cause_and_normalized_scoring_guidance() -> None: - prompt = _prompt(EvalAuthor.author_insight_metrics) - - assert "Focus the metric on the root cause, not the surface symptom" in prompt - assert "Every metric value must be a float in ``[0.0, 1.0]``" in prompt - assert "Error rate → ``max(0.0, 1.0 - errors / total_calls)``" in prompt - assert "Presence of a behavior → ``1.0`` if present, ``0.0`` if absent" in prompt - assert "Partial credit → fraction of required steps completed correctly" in prompt - assert "Do not hard-code scores for the production traces" in prompt - - -def test_eval_author_prompts_retain_template_path_and_harbor_name_guidance() -> None: - prompt = _prompt(EvalAuthor.fill_task_template) - - assert "``task_template.uri`` (file:// URI)" in prompt - assert "edit this directory in place and do not copy or rename it" in prompt - assert "Leave unfillable placeholders as-is" in prompt - assert "keep ``[task] name`` in ``org/name``" in prompt - assert "deterministically finalize the name and provenance" in prompt - - def _install_pipeline( monkeypatch: pytest.MonkeyPatch, - outcomes: Sequence[Diagnostic | BaseException], eval_author: EvalAuthor, + outcomes: Sequence[Diagnostic | BaseException], *, - materialized_dataset: Dataset | None = None, -) -> _PipelineCalls: - calls = _PipelineCalls( - fill_task_template=[], - analyzer_init=[], - analyzer_run=[], - discovered_datasets=[], - author_args=[], - suite_discards=0, + validation_errors: dict[str, list[str]] | None = None, + contract_failures: int = 0, +) -> _Calls: + events: list[str] = [] + failures = validation_errors or {} + train_dataset = _MaterializedDataset( + "train", + eval_author.experiment_dir / "datasets" / "train", + events, + failures.get("train"), ) - next_analyzer = 0 + validation_dataset = _MaterializedDataset( + "validation", + eval_author.experiment_dir / "datasets" / "validation", + events, + failures.get("validation"), + ) + materialized = _MaterializedDataset( + "insight-suite", + eval_author.experiment_dir / "insight-root" / "insight-suite", + events, + failures.get("insight"), + ) + calls = _Calls( + events=events, + filled_refs=[], + analyzed_refs=[], + diagnostics=[], + author_feedback=[], + train_dataset=train_dataset, + validation_dataset=validation_dataset, + insight_suite=materialized, + ) + analyzer_index = 0 + contract_attempt = 0 class FakeInsightSuite: def __init__(self, *, task_template: Task, **_: Any) -> None: self.task_template = task_template - self.staged: list[Any] = [] + self.root = eval_author.experiment_dir / "insight-root" + self.suite_dir = self.root / "insight-suite" def stage(self, trace_refs: list[str]) -> list[Any]: - self.staged = [ - type( - "StagedTask", - (), - {"trace_ref": trace_ref, "task": self.task_template, "result": None}, - )() - for trace_ref in trace_refs + return [ + SimpleNamespace(trace_ref=trace_ref, task=self.task_template, result=None) for trace_ref in trace_refs ] - return self.staged def validate(self, staged: Any) -> None: - staged.result = calls.fill_task_template[-1].result + staged.result = Task(id=f"task-{staged.trace_ref}", uri=f"file:///tasks/{staged.trace_ref}") def promote_local(self, trace_refs: list[str], staged_tasks: list[Any]) -> Dataset: assert trace_refs == [staged.trace_ref for staged in staged_tasks] - tasks = [staged.result for staged in staged_tasks] - if materialized_dataset is not None: - materialized_dataset.tasks = tasks - self.materialized_dataset = materialized_dataset - return materialized_dataset - self.materialized_dataset = Dataset(id="insight-suite", tasks=tasks) - return self.materialized_dataset - - def discard(self) -> None: - calls.suite_discards += 1 + materialized.tasks = [staged.result for staged in staged_tasks] + return materialized def record_analysis(self, statuses: dict[str, tuple[str, str | None]]) -> None: - pass + del statuses def finalize(self) -> SimpleNamespace: - identity = "sha256:" + "a" * 64 - scorer_identity = "sha256:" + "b" * 64 - self.materialized_dataset.metadata.update( - { - "insight_suite_identity": identity, - "insight_suite_scorer_identity": scorer_identity, - "insight_suite_task_hashes": { - task.id: { - "content_hash": "sha256:" + "c" * 64, - "verifier_hash": "sha256:" + "d" * 64, - } - for task in self.materialized_dataset.list_tasks() - }, - } - ) + calls.events.append("finalize") return SimpleNamespace( - dataset=self.materialized_dataset, - identity=identity, - scorer_identity=scorer_identity, + identity=f"sha256:{'a' * 64}", + path=self.suite_dir, + dataset=materialized, ) - class FillTaskTemplate: - async def __call__( - self, - trace_ref: str, - task_template: Task, - client: Any, - workspace: str, - ) -> Task: - result = Task(id=f"task-{trace_ref}", uri=f"file:///tasks/{trace_ref}") - calls.fill_task_template.append( - _FillTaskTemplateCall( - trace_ref=trace_ref, - task_template=task_template, - client=client, - workspace=workspace, - result=result, - ) - ) - return result + def discard(self) -> None: + calls.suite_discards += 1 class FakeTraceAnalyzer: def __init__(self, *, experiment_dir: Path, config: TraceAnalyzerConfig) -> None: - nonlocal next_analyzer - self._index = next_analyzer - next_analyzer += 1 - calls.analyzer_init.append(_AnalyzerInitCall(experiment_dir=experiment_dir, config=config)) + nonlocal analyzer_index + del experiment_dir, config + self.index = analyzer_index + analyzer_index += 1 async def run( self, @@ -257,24 +189,30 @@ async def run( client: Any, workspace: str, ) -> Diagnostic: - calls.analyzer_run.append( - _AnalyzerRunCall( - trial=trial, - task=task, - agent_path=agent_path, - insight=insight, - client=client, - workspace=workspace, - ) - ) - outcome = outcomes[self._index] + del task, agent_path, insight, client, workspace + ref = cast(str, trial.metadata["trace_ref"]) + calls.analyzed_refs.append(ref) + outcome = outcomes[self.index] if isinstance(outcome, BaseException): raise outcome return outcome + class FillTaskTemplate: + async def __call__( + self, + trace_ref: str, + task_template: Task, + client: Any, + workspace: str, + ) -> Task: + del task_template, client, workspace + calls.filled_refs.append(trace_ref) + return Task(id=f"task-{trace_ref}", uri=f"file:///tasks/{trace_ref}") + class DiscoverRunner: async def __call__(self, dataset: Dataset) -> str: - calls.discovered_datasets.append(dataset) + assert dataset is train_dataset + calls.events.append("discover") return "runner conventions" class AuthorInsightMetrics: @@ -282,512 +220,233 @@ async def __call__( self, insight: Insight, diagnostics: list[tuple[str, Diagnostic]], + authored_train_dataset: Dataset, + authored_validation_dataset: Dataset, insight_suite: Dataset, runner_conventions: str, validation_feedback: str | None = None, - ) -> str: - calls.author_args.append( - ( - insight, - diagnostics, - insight_suite, - runner_conventions, - validation_feedback, - ) + ) -> MetricAuthoringResult: + del insight, runner_conventions + assert authored_train_dataset is train_dataset + assert authored_validation_dataset is validation_dataset + assert insight_suite is materialized + calls.events.append("author") + calls.diagnostics = diagnostics + calls.author_feedback.append(validation_feedback) + return MetricAuthoringResult( + metric_keys=("uses_correct_tool",), + summary="Authored tool-use metric.", ) - return "authored insight metrics" + + def validate_metric_contracts( + datasets: dict[str, Dataset], + *, + metric_keys: tuple[str, ...], + ) -> None: + nonlocal contract_attempt + assert datasets == { + "train": train_dataset, + "validation": validation_dataset, + "insight": materialized, + } + assert metric_keys == ("uses_correct_tool",) + calls.events.append("contract") + contract_attempt += 1 + if contract_attempt <= contract_failures: + raise DatasetValidationError("validation/task-a: metric-contract.json declares the wrong keys") eval_author.fill_task_template = cast(Any, FillTaskTemplate()) eval_author.discover_runner = cast(Any, DiscoverRunner()) eval_author.author_insight_metrics = cast(Any, AuthorInsightMetrics()) - monkeypatch.setattr(eval_author_module, "TraceAnalyzer", FakeTraceAnalyzer) monkeypatch.setattr(eval_author_module, "InsightSuite", FakeInsightSuite) + monkeypatch.setattr(eval_author_module, "TraceAnalyzer", FakeTraceAnalyzer) + monkeypatch.setattr(eval_author_module, "validate_metric_contracts", validate_metric_contracts) + + monkeypatch.setattr(eval_author_module.cache, "store", lambda *args: None) + monkeypatch.setattr(eval_author_module, "doc", lambda *_args, **_kwargs: "dataset docs") return calls +def test_metric_authoring_prompt_limits_edits_and_result_shape() -> None: + prompt = " ".join((inspect.getdoc(EvalAuthor.author_insight_metrics) or "").split()) + + assert "every task in ``train_dataset``, ``validation_dataset``, and ``insight_suite``" in prompt + assert "task-specific verifier edits" in prompt.lower() + assert "Do not hard-code scores for the production traces" in prompt + assert "metric-contract.json" in prompt + assert "MetricAuthoringResult" in prompt + assert "metric_keys" in prompt + assert "verifier_bundle" not in prompt + + @pytest.mark.asyncio -async def test_run_without_traces_returns_input_datasets_unchanged(tmp_path: Path) -> None: +async def test_run_returns_input_datasets_without_trace_refs( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: eval_author = _eval_author(tmp_path) - train_dataset = Dataset(id="train") - validation_dataset = Dataset(id="validation") + calls = _install_pipeline(monkeypatch, eval_author, []) result = await eval_author.run( _insight([]), Path("agent"), Task(id="template"), - train_dataset, - validation_dataset, + calls.train_dataset, + calls.validation_dataset, client=cast(Any, object()), ) - assert result.train_dataset is train_dataset - assert result.validation_dataset is validation_dataset - assert result.summary == "No trace refs on insight — nothing to analyze." + assert result.train_dataset is calls.train_dataset + assert result.validation_dataset is calls.validation_dataset + assert result.insight_suite is None + assert result.insight_suite_identity is None + assert result.metric_keys == () assert cast(_ClosingShell, eval_author.shell).close_calls == 1 @pytest.mark.asyncio -async def test_run_requires_persisted_insight_id(tmp_path: Path) -> None: - insight = Insight( - workspace="workspace-a", - title="Tool selection failures", - description="The agent selects the wrong tool.", - agent="research-agent", - trace_refs=["trace-1"], - ) - - with pytest.raises(ValueError, match="persisted Insight with a durable id"): - await _eval_author(tmp_path).run( - insight, - Path("agent"), - Task(id="template"), - Dataset(id="train"), - Dataset(id="validation"), - client=cast(Any, object()), - ) - - -@pytest.mark.asyncio -async def test_run_enforces_max_traces( +async def test_run_materializes_authors_validates_and_returns_datasets( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - diagnostics = [_diagnostic("one"), _diagnostic("two")] eval_author = _eval_author(tmp_path, max_traces=2) - calls = _install_pipeline(monkeypatch, diagnostics, eval_author) - monkeypatch.setattr(eval_author_module.cache, "store", lambda *args: None) + diagnostic = _diagnostic() + calls = _install_pipeline(monkeypatch, eval_author, [diagnostic, diagnostic]) - await eval_author.run( - _insight(["trace-1", "trace-2", "trace-3"]), + result = await eval_author.run( + _insight(["trace-1", "trace-2", "ignored"]), Path("agent"), Task(id="template"), - Dataset(id="train"), - Dataset(id="validation"), + calls.train_dataset, + calls.validation_dataset, client=cast(Any, object()), ) - assert [call.trace_ref for call in calls.fill_task_template] == ["trace-1", "trace-2"] - assert [call.trial.metadata["trace_ref"] for call in calls.analyzer_run] == ["trace-1", "trace-2"] - - -@pytest.mark.asyncio -async def test_run_discards_staged_suite_when_filling_fails( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - eval_author = _eval_author(tmp_path) - calls = _install_pipeline(monkeypatch, [], eval_author) - - class FailedFillTaskTemplate: - async def __call__(self, *_: Any) -> Task: - raise RuntimeError("fill failed") - - eval_author.fill_task_template = cast(Any, FailedFillTaskTemplate()) - - with pytest.raises(RuntimeError, match="fill failed"): - await eval_author.run( - _insight(["trace-1"]), - Path("agent"), - Task(id="template"), - Dataset(id="train"), - Dataset(id="validation"), - client=cast(Any, object()), - ) - - assert calls.suite_discards == 1 - assert cast(_ClosingShell, eval_author.shell).close_calls == 1 - - -def test_trace_trial_uses_intake_uri_and_insight_metadata(tmp_path: Path) -> None: - insight = _insight(["trace-7"]) - - trial = _eval_author(tmp_path)._trace_trial(insight, Task(id="task-7"), "trace-7", 3, insight.id) - - assert trial.id == "insight-trace-3" - assert trial.task_id == "task-7" - assert trial.status == "completed" - assert trial.trace is not None - assert trial.trace.uri == "intake://trace-7" - assert trial.trace.description == "Production trace attached to the insight." - assert trial.metadata == { - "source": "insight", - "trace_ref": "trace-7", - "insight_id": insight.id, - } + assert calls.filled_refs == ["trace-1", "trace-2"] + assert calls.analyzed_refs == ["trace-1", "trace-2"] + assert calls.diagnostics == [("trace-1", diagnostic), ("trace-2", diagnostic)] + assert calls.events == [ + "discover", + "author", + "validate:train", + "validate:validation", + "validate:insight-suite", + "contract", + "finalize", + ] + assert result.train_dataset is calls.train_dataset + assert result.validation_dataset is calls.validation_dataset + assert result.insight_suite is calls.insight_suite + assert result.insight_suite_identity == f"sha256:{'a' * 64}" + assert result.metric_keys == ("uses_correct_tool",) + assert result.summary == "Authored tool-use metric." @pytest.mark.asyncio -async def test_run_caches_each_successful_diagnostic( +async def test_metric_contract_failure_uses_repair_loop( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - first = _diagnostic("first") - second = _diagnostic("second") - eval_author = _eval_author(tmp_path) - _install_pipeline(monkeypatch, [first, second], eval_author) - stored: list[tuple[Path, str, Diagnostic]] = [] - monkeypatch.setattr(eval_author_module.cache, "store", lambda *args: stored.append(args)) + eval_author = _eval_author(tmp_path, max_validation_repair_attempts=1) + calls = _install_pipeline( + monkeypatch, + eval_author, + [_diagnostic()], + contract_failures=1, + ) await eval_author.run( - _insight(["trace-a", "trace-b"]), + _insight(["trace-1"]), Path("agent"), Task(id="template"), - Dataset(id="train"), - Dataset(id="validation"), + calls.train_dataset, + calls.validation_dataset, client=cast(Any, object()), ) - assert stored == [ - (tmp_path, eval_author_module.cache.task_hash("eval_author:trace-a"), first), - (tmp_path, eval_author_module.cache.task_hash("eval_author:trace-b"), second), + assert calls.author_feedback == [ + None, + "validation/task-a: metric-contract.json declares the wrong keys", ] + assert calls.events.count("author") == 2 + assert calls.events.count("contract") == 2 @pytest.mark.asyncio -async def test_run_skips_failed_trace_analysis_and_keeps_successes( +async def test_static_validation_failure_uses_same_repair_loop( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, ) -> None: - successful = _diagnostic("successful") - eval_author = _eval_author(tmp_path) - calls = _install_pipeline(monkeypatch, [RuntimeError("analysis failed"), successful], eval_author) - stored: list[Diagnostic] = [] - monkeypatch.setattr(eval_author_module.cache, "store", lambda workspace, key, value: stored.append(value)) - insight = _insight(["trace-bad", "trace-good"]) + eval_author = _eval_author(tmp_path, max_validation_repair_attempts=1) + calls = _install_pipeline( + monkeypatch, + eval_author, + [_diagnostic()], + validation_errors={"validation": ["task 'task-a': tests/check.py: invalid syntax"]}, + ) - result = await eval_author.run( - insight, + await eval_author.run( + _insight(["trace-1"]), Path("agent"), Task(id="template"), - Dataset(id="train"), - Dataset(id="validation"), + calls.train_dataset, + calls.validation_dataset, client=cast(Any, object()), ) - assert result.summary == "authored insight metrics" - assert stored == [successful] - assert calls.author_args[0][1] == [("trace-good", successful)] - assert "Trace analysis failed for trace-bad: analysis failed" in caplog.text + assert calls.author_feedback[0] is None + assert "validation" in cast(str, calls.author_feedback[1]) + assert "invalid syntax" in cast(str, calls.author_feedback[1]) + assert calls.events.count("author") == 2 + assert calls.events.count("validate:train") == 2 + assert calls.events.count("validate:validation") == 2 + assert calls.events.count("validate:insight-suite") == 2 + assert calls.events.count("contract") == 1 @pytest.mark.asyncio -async def test_run_propagates_trace_analysis_cancellation( +async def test_run_discards_staged_tasks_and_closes_shell_when_fill_fails( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: eval_author = _eval_author(tmp_path) - calls = _install_pipeline(monkeypatch, [asyncio.CancelledError()], eval_author) - monkeypatch.setattr(eval_author_module.cache, "store", lambda *args: None) + calls = _install_pipeline(monkeypatch, eval_author, []) - with pytest.raises(asyncio.CancelledError): + class FailFill: + async def __call__(self, *_: object) -> Task: + raise RuntimeError("fill failed") + + eval_author.fill_task_template = cast(Any, FailFill()) + with pytest.raises(RuntimeError, match="fill failed"): await eval_author.run( - _insight(["trace-cancelled"]), + _insight(["trace-1"]), Path("agent"), Task(id="template"), - Dataset(id="train"), - Dataset(id="validation"), + calls.train_dataset, + calls.validation_dataset, client=cast(Any, object()), ) - assert calls.author_args == [] - - -@pytest.mark.asyncio -async def test_run_authors_metrics_on_materialized_insight_suite( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - diagnostic = _diagnostic("diagnostic") - eval_author = _eval_author(tmp_path, max_summary_tokens=12_345) - calls = _install_pipeline(monkeypatch, [diagnostic], eval_author) - monkeypatch.setattr(eval_author_module.cache, "store", lambda *args: None) - documentation = object() - doc_calls: list[tuple[type[Dataset], int]] = [] - - def fake_doc(dataset_type: type[Dataset], *, inline_depth: int) -> object: - doc_calls.append((dataset_type, inline_depth)) - return documentation - - monkeypatch.setattr(eval_author_module, "doc", fake_doc) - insight = _insight(["trace-1"]) - task_template = Task(id="template") - train_dataset = Dataset(id="train") - validation_dataset = Dataset(id="validation") - client = cast(Any, object()) - - result = await eval_author.run( - insight, - Path("relative-agent"), - task_template, - train_dataset, - validation_dataset, - client=client, - ) - - assert result.summary == "authored insight metrics" - assert len(calls.fill_task_template) == 1 - fill_call = calls.fill_task_template[0] - assert fill_call.trace_ref == "trace-1" - assert fill_call.task_template is task_template - assert fill_call.client is client - assert fill_call.workspace == "workspace-a" - assert fill_call.result == Task(id="task-trace-1", uri="file:///tasks/trace-1") - - assert len(calls.analyzer_init) == 1 - assert calls.analyzer_init[0].experiment_dir == tmp_path - assert type(calls.analyzer_init[0].config) is TraceAnalyzerConfig - assert calls.analyzer_init[0].config.max_summary_tokens == 12_345 - - assert len(calls.analyzer_run) == 1 - analyzer_call = calls.analyzer_run[0] - assert analyzer_call.trial == eval_author._trace_trial( - insight, - fill_call.result, - "trace-1", - 1, - insight.id, - ) - assert analyzer_call.task is fill_call.result - assert analyzer_call.agent_path == tmp_path / "relative-agent" - assert analyzer_call.insight is insight - assert analyzer_call.client is client - assert analyzer_call.workspace == "workspace-a" - - assert doc_calls == [(Dataset, 1)] - assert eval_author.context["dataset_documentation"] is documentation - assert len(calls.discovered_datasets) == 1 - materialized_dataset = calls.discovered_datasets[0] - assert result.insight_suite is materialized_dataset - assert result.insight_suite_identity == f"sha256:{'a' * 64}" - assert materialized_dataset.id == "insight-suite" - assert materialized_dataset is not train_dataset - assert materialized_dataset is not validation_dataset - assert len(calls.author_args) == 1 - ( - authored_insight, - diagnostics, - authored_suite, - runner_conventions, - validation_feedback, - ) = calls.author_args[0] - assert authored_insight is insight - assert diagnostics == [("trace-1", diagnostic)] - assert authored_suite is materialized_dataset - assert runner_conventions == "runner conventions" - assert validation_feedback is None - assert result.train_dataset is train_dataset - assert result.validation_dataset is validation_dataset - - -class _RepairableDataset(Dataset): - def __init__(self, id: str, error: str | None) -> None: - super().__init__(id=id) - self.error = error - self.validate_calls = 0 - - async def validate(self) -> None: - self.validate_calls += 1 - if self.error is not None: - raise DatasetValidationError(self.error) + assert calls.suite_discards == 1 + assert cast(_ClosingShell, eval_author.shell).close_calls == 1 @pytest.mark.asyncio -async def test_run_feeds_validation_failures_back_for_one_repair_attempt( +async def test_run_propagates_trace_analysis_cancellation( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: eval_author = _eval_author(tmp_path) - insight_dataset = _RepairableDataset("insight-suite", "task 'insight-a': check.py:2:1: invalid syntax") - _install_pipeline( - monkeypatch, - [_diagnostic("diagnostic")], - eval_author, - materialized_dataset=insight_dataset, - ) - monkeypatch.setattr(eval_author_module.cache, "store", lambda *args: None) - train_dataset = Dataset(id="train") - validation_dataset = Dataset(id="validation") - client = cast(Any, object()) - feedback: list[str | None] = [] - - class RepairInsightMetrics: - async def __call__( - self, - insight: Insight, - diagnostics: list[tuple[str, Diagnostic]], - insight_suite: Dataset, - runner_conventions: str, - validation_feedback: str | None = None, - ) -> str: - feedback.append(validation_feedback) - if validation_feedback is not None: - insight_dataset.error = None - return "repaired insight metric" - - eval_author.author_insight_metrics = cast(Any, RepairInsightMetrics()) - - result = await eval_author.run( - _insight(["trace-1"]), - Path("agent"), - Task(id="template"), - train_dataset, - validation_dataset, - client=client, - ) - - assert result.train_dataset is train_dataset - assert result.validation_dataset is validation_dataset - assert feedback[0] is None - assert feedback[1] is not None - assert "task 'insight-a': check.py:2:1: invalid syntax" in feedback[1] - assert insight_dataset.validate_calls == 2 - - -@pytest.mark.asyncio -async def test_run_raises_after_validation_repair_budget_is_exhausted( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - eval_author = _eval_author(tmp_path, max_validation_repair_attempts=1) - insight_dataset = _RepairableDataset("insight-suite", "task 'insight-a': check.py:2:1: invalid syntax") - calls = _install_pipeline( - monkeypatch, - [_diagnostic("diagnostic")], - eval_author, - materialized_dataset=insight_dataset, - ) - monkeypatch.setattr(eval_author_module.cache, "store", lambda *args: None) + calls = _install_pipeline(monkeypatch, eval_author, [asyncio.CancelledError()]) - with pytest.raises(DatasetValidationError) as exc_info: + with pytest.raises(asyncio.CancelledError): await eval_author.run( _insight(["trace-1"]), Path("agent"), Task(id="template"), - Dataset(id="train"), - Dataset(id="validation"), + calls.train_dataset, + calls.validation_dataset, client=cast(Any, object()), ) - assert "task 'insight-a': check.py:2:1: invalid syntax" in str(exc_info.value) - assert len(calls.author_args) == 2 - assert calls.author_args[0][-1] is None - assert calls.author_args[1][-1] is not None - assert insight_dataset.validate_calls == 2 - - -def test_eval_author_config_defaults_and_bounds_validation_repair_attempts() -> None: - # Experimentalist's loop config asserts this default too, but from the other side of - # the plugin boundary; owning it here is what keeps the default a plugin contract. - assert EvalAuthorConfig().max_validation_repair_attempts == 5 - assert EvalAuthorConfig(max_validation_repair_attempts=10).max_validation_repair_attempts == 10 - with pytest.raises(ValueError, match="less than or equal to 10"): - EvalAuthorConfig(max_validation_repair_attempts=11) - - -def _string_reporter() -> tuple[Any, io.StringIO]: - from nemo_experimentalist_plugin.experimentalist.reporting import RunReporter - - sink = io.StringIO() - return RunReporter(sink=sink), sink - - -@pytest.mark.asyncio -async def test_run_with_reporter_emits_start_note_and_complete_on_empty_traces(tmp_path: Path) -> None: - reporter, sink = _string_reporter() - eval_author = _eval_author(tmp_path, reporter=reporter) - - await eval_author.run( - _insight([]), - Path("agent"), - Task(id="template"), - Dataset(id="train"), - Dataset(id="validation"), - client=cast(Any, object()), - ) - - out = sink.getvalue() - assert "eval author · starting" in out - assert "no trace refs — nothing to analyze" in out - assert "eval author · complete" in out - - -@pytest.mark.asyncio -async def test_run_with_reporter_emits_pipeline_phases( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - reporter, sink = _string_reporter() - eval_author = _eval_author(tmp_path, reporter=reporter) - _install_pipeline( - monkeypatch, - [_diagnostic("trace-1 failed"), _diagnostic("trace-2 failed")], - eval_author, - ) - monkeypatch.setattr(eval_author_module.cache, "store", lambda *args: None) - - result = await eval_author.run( - _insight(["trace-1", "trace-2"]), - Path("agent"), - Task(id="template"), - Dataset(id="train"), - Dataset(id="validation"), - client=cast(Any, object()), - ) - - out = sink.getvalue() - assert "eval author · starting" in out - assert "eval author · materializing tasks" in out - assert "task 2/≤2" in out - assert "eval author · analyzing traces" in out - assert "eval author · discovering runner" in out - assert "eval author · authoring metrics" in out - assert "eval author · complete" in out - assert "Finished" not in out - assert result.summary == "authored insight metrics" - - -@pytest.mark.asyncio -async def test_run_with_reporter_emits_repair_progress( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - reporter, sink = _string_reporter() - eval_author = _eval_author(tmp_path, max_validation_repair_attempts=2, reporter=reporter) - insight_dataset = _RepairableDataset("insight-suite", "task 'insight-a': check.py:2:1: invalid syntax") - _install_pipeline( - monkeypatch, - [_diagnostic("diagnostic")], - eval_author, - materialized_dataset=insight_dataset, - ) - monkeypatch.setattr(eval_author_module.cache, "store", lambda *args: None) - - class RepairInsightMetrics: - async def __call__( - self, - insight: Insight, - diagnostics: list[tuple[str, Diagnostic]], - insight_suite: Dataset, - runner_conventions: str, - validation_feedback: str | None = None, - ) -> str: - if validation_feedback is not None: - insight_dataset.error = None - return "repaired insight metric" - - eval_author.author_insight_metrics = cast(Any, RepairInsightMetrics()) - - await eval_author.run( - _insight(["trace-1"]), - Path("agent"), - Task(id="template"), - Dataset(id="train"), - Dataset(id="validation"), - client=cast(Any, object()), - ) - - out = sink.getvalue() - assert "eval author · repairing metrics" in out - assert "attempt 1/≤2" in out - assert "eval author · complete" in out + assert "author" not in calls.events diff --git a/plugins/nemo-eval-author/tests/test_eval_author_materialization.py b/plugins/nemo-eval-author/tests/test_eval_author_materialization.py index ca4cf7d17b..cfead4caaa 100644 --- a/plugins/nemo-eval-author/tests/test_eval_author_materialization.py +++ b/plugins/nemo-eval-author/tests/test_eval_author_materialization.py @@ -11,8 +11,8 @@ import pytest from nemo_eval_author_plugin.eval_author import materialization as materialization_module -from nemo_eval_author_plugin.eval_author.materialization import InsightSuite -from nemo_experimentalist_plugin.entities import Task +from nemo_eval_author_plugin.eval_author.materialization import InsightSuite, validate_metric_contracts +from nemo_experimentalist_plugin.entities import Dataset, DatasetValidationError, Task from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import HarborDataset @@ -46,6 +46,65 @@ def _write_template(root: Path) -> Task: return Task(id="task-template", uri=root.as_uri()) +def _dataset_with_metric_contract( + root: Path, + dataset_id: str, + payload: dict[str, object] | str | None, +) -> Dataset: + task = _write_template(root / dataset_id / "task") + if payload is not None: + content = payload if isinstance(payload, str) else json.dumps(payload) + (root / dataset_id / "task" / "tests" / "metric-contract.json").write_text(content, encoding="utf-8") + return Dataset(id=dataset_id, tasks=[task]) + + +def test_metric_contracts_require_the_same_authored_key_set_across_datasets(tmp_path: Path) -> None: + datasets = { + "train": _dataset_with_metric_contract( + tmp_path, + "train", + {"metric_keys": ["uses_required_tool", "cites_source"]}, + ), + "validation": _dataset_with_metric_contract( + tmp_path, + "validation", + {"metric_keys": ["cites_source", "uses_required_tool"]}, + ), + "insight": _dataset_with_metric_contract( + tmp_path, + "insight", + {"metric_keys": ["uses_required_tool", "cites_source"]}, + ), + } + + validate_metric_contracts( + datasets, + metric_keys=("uses_required_tool", "cites_source"), + ) + + +@pytest.mark.parametrize( + ("payload", "message"), + [ + (None, "missing"), + ("not json", "invalid JSON"), + ({"metric_keys": ["different_key"]}, "expected"), + ], +) +def test_metric_contracts_reject_missing_malformed_or_mismatched_keys( + tmp_path: Path, + payload: dict[str, object] | str | None, + message: str, +) -> None: + dataset = _dataset_with_metric_contract(tmp_path, "train", payload) + + with pytest.raises(DatasetValidationError, match=message): + validate_metric_contracts( + {"train": dataset}, + metric_keys=("uses_required_tool",), + ) + + def test_insight_suite_materializes_discoverable_tasks_with_provenance(tmp_path: Path) -> None: template = _write_template(tmp_path / "template") refs = ["intake/traces/unsafe ref", "intake/traces/unsafe ref"] diff --git a/plugins/nemo-eval-author/tests/test_eval_author_models.py b/plugins/nemo-eval-author/tests/test_eval_author_models.py new file mode 100644 index 0000000000..554d1c7a5f --- /dev/null +++ b/plugins/nemo-eval-author/tests/test_eval_author_models.py @@ -0,0 +1,138 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Minimal public models for Eval Author artifacts.""" + +from pathlib import Path + +import pytest +from nemo_eval_author_plugin.eval_author import models +from nemo_eval_author_plugin.eval_author.models import EvalAuthorResult, MetricAuthoringResult +from nemo_experimentalist_plugin.entities import Dataset +from pydantic import ValidationError + + +def _dataset(name: str) -> Dataset: + return Dataset(id=name) + + +def test_overdesigned_public_models_and_inventory_module_are_absent() -> None: + removed_models = { + "AuthoredMetric", + "AuthoredMetricContract", + "ArtifactDescriptor", + "EvalAuthorEvaluationContext", + "EvalAuthorRequest", + "FrozenJsonObject", + "InsightRef", + "ReadOnlyDatasetRef", + } + + assert removed_models.isdisjoint(vars(models)) + assert not (Path(models.__file__).with_name("inventory.py")).exists() + + +def test_result_returns_the_same_dataset_objects() -> None: + authored = MetricAuthoringResult( + metric_keys=("uses_correct_tool",), + summary="Added a tool-use metric.", + ) + train_dataset = _dataset("train") + validation_dataset = _dataset("validation") + insight_suite = _dataset("insight") + result = EvalAuthorResult( + train_dataset=train_dataset, + validation_dataset=validation_dataset, + insight_suite=insight_suite, + insight_suite_identity=f"sha256:{'a' * 64}", + metric_keys=authored.metric_keys, + summary=authored.summary, + ) + + assert set(MetricAuthoringResult.model_fields) == {"metric_keys", "summary"} + assert set(EvalAuthorResult.model_fields) == { + "train_dataset", + "validation_dataset", + "insight_suite", + "insight_suite_identity", + "metric_keys", + "summary", + } + assert result.train_dataset is train_dataset + assert result.validation_dataset is validation_dataset + assert result.insight_suite is insight_suite + + +@pytest.mark.parametrize( + "metric_keys", + [ + (), + ("",), + ("uses_correct_tool", "uses_correct_tool"), + ("reward", "score"), + ], +) +def test_metric_authoring_requires_unique_non_generic_keys(metric_keys: tuple[str, ...]) -> None: + with pytest.raises(ValidationError): + MetricAuthoringResult(metric_keys=metric_keys, summary="Authored metrics.") + + +def test_eval_author_result_supports_unchanged_datasets_without_authored_tasks() -> None: + result = EvalAuthorResult( + train_dataset=_dataset("train"), + validation_dataset=_dataset("validation"), + summary="No trace refs on insight.", + ) + + assert result.insight_suite is None + assert result.insight_suite_identity is None + assert result.metric_keys == () + assert result.summary == "No trace refs on insight." + + +def test_eval_author_result_stores_normalized_metric_keys() -> None: + result = EvalAuthorResult( + train_dataset=_dataset("train"), + validation_dataset=_dataset("validation"), + insight_suite=_dataset("insight"), + insight_suite_identity=f"sha256:{'a' * 64}", + metric_keys=(" uses_correct_tool ",), + summary="Normalized keys.", + ) + + assert result.metric_keys == ("uses_correct_tool",) + + +def test_eval_author_result_requires_suite_identity_and_declared_metrics_together() -> None: + with pytest.raises(ValidationError): + EvalAuthorResult( + train_dataset=_dataset("train"), + validation_dataset=_dataset("validation"), + metric_keys=("uses_correct_tool",), + summary="Incomplete.", + ) + with pytest.raises(ValidationError): + EvalAuthorResult( + train_dataset=_dataset("train"), + validation_dataset=_dataset("validation"), + insight_suite=_dataset("insight"), + metric_keys=("uses_correct_tool",), + summary="Missing identity.", + ) + with pytest.raises(ValidationError): + EvalAuthorResult( + train_dataset=_dataset("train"), + validation_dataset=_dataset("validation"), + insight_suite=_dataset("insight"), + insight_suite_identity=f"sha256:{'a' * 64}", + metric_keys=("reward",), + summary="Generic only.", + ) + with pytest.raises(ValidationError): + EvalAuthorResult( + train_dataset=_dataset("train"), + validation_dataset=_dataset("validation"), + insight_suite=_dataset("insight"), + insight_suite_identity=f"sha256:{'a' * 64}", + summary="Missing metric keys.", + ) diff --git a/plugins/nemo-eval-author/tests/test_eval_author_repair_e2e.py b/plugins/nemo-eval-author/tests/test_eval_author_repair_e2e.py index 4ca01f081e..cb67f8b258 100644 --- a/plugins/nemo-eval-author/tests/test_eval_author_repair_e2e.py +++ b/plugins/nemo-eval-author/tests/test_eval_author_repair_e2e.py @@ -6,6 +6,7 @@ import asyncio import json import os +import shlex from pathlib import Path import pytest @@ -73,6 +74,44 @@ def check_tool_hallucination() -> float: } ] } +_KNOWN_COMPLIANT_TRACE = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "name": "inventory_lookup", + "attributes": [ + { + "key": "openinference.span.kind", + "value": {"stringValue": "TOOL"}, + }, + { + "key": "output.value", + "value": {"stringValue": '{"warehouse":"Denver","available_units":8}'}, + }, + ], + }, + { + "name": "generate_response", + "attributes": [ + { + "key": "openinference.span.kind", + "value": {"stringValue": "CHAIN"}, + }, + { + "key": "output.value", + "value": {"stringValue": "The Denver warehouse has eight units available."}, + }, + ], + }, + ] + } + ] + } + ] +} def _write_malformed_task(dataset_dir: Path, task_id: str) -> None: @@ -150,22 +189,39 @@ def _write_known_failing_task(dataset_dir: Path) -> None: def _write_known_failing_agent(agent_dir: Path) -> None: + _write_harbor_trace_agent( + agent_dir, + agent_name="known-failing-baseline", + trace_payload=_KNOWN_FAILING_TRACE, + ) + + +def _write_harbor_trace_agent( + agent_dir: Path, + *, + agent_name: str, + trace_payload: object | None, +) -> None: agent_dir.mkdir() - trace_payload = json.dumps(_KNOWN_FAILING_TRACE, separators=(",", ":")) + serialized_trace = json.dumps(trace_payload, separators=(",", ":")) if trace_payload is not None else None + trace_command = ( + "mkdir -p /logs/artifacts/traces && " + f"printf '%s\\n' {shlex.quote(serialized_trace)} > /logs/artifacts/traces/trace.jsonl" + if serialized_trace is not None + else "true" + ) (agent_dir / "harbor_wrapper.py").write_text( f"""\ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import shlex - from harbor import AgentContext, BaseAgent, BaseEnvironment class WrappedAgent(BaseAgent): @staticmethod def name() -> str: - return "known-failing-baseline" + return {agent_name!r} def version(self) -> str: return "1.0.0" @@ -179,12 +235,7 @@ async def run( environment: BaseEnvironment, context: AgentContext, ) -> None: - trace_payload = {trace_payload!r} - command = ( - "mkdir -p /logs/artifacts/traces && " - f"printf '%s\\\\n' {{shlex.quote(trace_payload)}} " - "> /logs/artifacts/traces/trace.jsonl" - ) + command = {trace_command!r} process = await environment.exec(command) context.metadata = {{ "instruction": instruction, @@ -244,19 +295,20 @@ async def test_gpt5_mini_repairs_malformed_harbor_verifiers( "Python verifier files are statically checked by await dataset.validate(); test.sh is checked as Bash. " "Preserve the existing verifier's intended behavior and repair every validation error." ) - summary = await asyncio.wait_for( eval_author.author_insight_metrics( insight, [], insight_suite, + insight_suite, + insight_suite, runner_conventions, validation_feedback=validation_feedback, ), timeout=300, ) - assert summary + assert summary.summary await insight_suite.validate() for verifier_path in ( insight_suite_dir / "task-a" / "tests" / "check_tool_hallucination.py", @@ -323,18 +375,19 @@ async def test_eval_author_metric_scores_known_failing_harbor_baseline_low( "insight_suite.validate() once after editing, then return the metric summary as soon as validation passes; " "do not inspect unrelated files." ) - summary = await asyncio.wait_for( eval_author.author_insight_metrics( insight, [("known-failing-trace", diagnostic)], insight_suite, + insight_suite, + insight_suite, runner_conventions, ), timeout=600, ) - assert summary + assert summary.summary await insight_suite.validate() evaluator = HarborEvaluator(experiment_dir=tmp_path) result = await asyncio.wait_for( @@ -371,12 +424,163 @@ async def test_eval_author_metric_scores_known_failing_harbor_baseline_low( assert all(0.0 <= value <= 1.0 for value in insight_metric_values.values()) assert min(insight_metric_values.values()) <= 0.25 assert insight_metric_names <= set(trial.metrics) + assert set(summary.metric_keys) == insight_metric_names print( json.dumps( { - "authored_summary": summary, + "authored_summary": summary.model_dump(mode="json"), "reward_json": reward_payload, }, sort_keys=True, ) ) + + +@pytest.mark.skipif( + not (_RUN_EVAL_AUTHOR_HARBOR_E2E and _HAS_LLM), + reason=f"Set RUN_EVAL_AUTHOR_HARBOR_E2E=1 with {_CREDENTIALS_HINT} to run the live Harbor metric canary.", +) +async def test_eval_author_metric_discriminates_controlled_harbor_tool_evidence( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An authored metric distinguishes measurable violations from compliant tool evidence.""" + insight_suite_dir = tmp_path / "insight-suite" + violating_agent_dir = tmp_path / "violating-agent" + compliant_agent_dir = tmp_path / "compliant-agent" + unmeasurable_agent_dir = tmp_path / "unmeasurable-agent" + _write_known_failing_task(insight_suite_dir) + _write_harbor_trace_agent( + violating_agent_dir, + agent_name="controlled-tool-violation", + trace_payload=_KNOWN_FAILING_TRACE, + ) + _write_harbor_trace_agent( + compliant_agent_dir, + agent_name="controlled-tool-compliance", + trace_payload=_KNOWN_COMPLIANT_TRACE, + ) + _write_harbor_trace_agent( + unmeasurable_agent_dir, + agent_name="controlled-unmeasurable-trace", + trace_payload=None, + ) + insight_suite = HarborDataset.from_path(insight_suite_dir) + await insight_suite.validate() + + llm = get_fast_model() + llm.config["temperature"] = 0.0 + monkeypatch.delenv("NEMO_EXPERIMENTALIST_API_KEY", raising=False) + eval_author = EvalAuthor( + experiment_dir=tmp_path, + config=EvalAuthorConfig(), + llm=llm, + ) + eval_author.context.pop("trace_documentation", None) + insight = Insight( + workspace="local", + title="Agent must use inventory_lookup before reporting availability", + description=( + "The agent must call inventory_lookup before reporting the Denver warehouse availability. " + "Measure current execution evidence of that tool call, not answer text or static task files." + ), + agent="controlled-tool-evidence", + ) + diagnostic = Diagnostic( + outcome="FAILURE", + summary=( + "The violating trace has a response span but no inventory_lookup tool span, so the absence " + "of the required call is measurable." + ), + failure_point=1, + root_cause="The agent answered from memory without calling inventory_lookup.", + ) + runner_conventions = ( + "This is a Harbor dataset. Preserve the existing reward metric and add a numeric root-cause metric to " + "/logs/verifier/reward.json. Higher values are better and values are bounded to [0.0, 1.0]. A readable " + "trace with no inventory_lookup tool span is measurable failing evidence and should score low. A readable " + "trace with an inventory_lookup tool span before the answer is compliant evidence and should score high. " + "A missing, unreadable, or malformed trace is not measurable evidence: fail the verifier with a clear " + "error instead of writing a fabricated 0.0 metric. Use a Python standard-library checker, avoid " + "unguarded grep under set -e, make the minimal verifier-only edit, validate the dataset once, then stop." + ) + summary = await asyncio.wait_for( + eval_author.author_insight_metrics( + insight, + [("controlled-tool-violation", diagnostic)], + insight_suite, + insight_suite, + insight_suite, + runner_conventions, + ), + timeout=600, + ) + + assert summary.summary + await insight_suite.validate() + evaluator = HarborEvaluator(experiment_dir=tmp_path) + + async def run_agent(agent_dir: Path, job_name: str): + return await asyncio.wait_for( + evaluator.run( + agent=agent_dir, + dataset=insight_suite, + options=HarborEvaluatorConfig( + force_rerun=True, + job_name=job_name, + jobs_dir=Path("harbor-jobs"), + n_concurrent_trials=1, + quiet=True, + ), + ), + timeout=600, + ) + + violating_result = await run_agent(violating_agent_dir, "controlled-tool-violation") + compliant_result = await run_agent(compliant_agent_dir, "controlled-tool-compliance") + unmeasurable_result = await run_agent(unmeasurable_agent_dir, "controlled-unmeasurable-trace") + + def reward_payload(result) -> dict[str, object]: + assert len(result.trials) == 1 + trial = result.trials[0] + assert trial.status == "completed", trial.error + reward_ref = trial.resources["log:verifier/reward.json"] + reward_path = local_path_from_uri(reward_ref.uri, context="Harbor verifier reward") + return json.loads(reward_path.read_text(encoding="utf-8")) + + violating_payload = reward_payload(violating_result) + compliant_payload = reward_payload(compliant_result) + assert violating_payload["reward"] == pytest.approx(1.0) + assert compliant_payload["reward"] == pytest.approx(1.0) + violating_metric_names = set(violating_payload) - {"reward"} + compliant_metric_names = set(compliant_payload) - {"reward"} + assert violating_metric_names + assert compliant_metric_names == violating_metric_names + assert set(summary.metric_keys) == violating_metric_names + for metric_name in sorted(violating_metric_names): + violating_score = violating_payload[metric_name] + compliant_score = compliant_payload[metric_name] + assert isinstance(violating_score, int | float) and not isinstance(violating_score, bool) + assert isinstance(compliant_score, int | float) and not isinstance(compliant_score, bool) + assert 0.0 <= violating_score <= 1.0 + assert 0.0 <= compliant_score <= 1.0 + assert compliant_score > violating_score + + assert len(unmeasurable_result.trials) == 1 + unmeasurable_trial = unmeasurable_result.trials[0] + assert unmeasurable_trial.status != "completed" + verifier_stderr_ref = unmeasurable_trial.resources["log:verifier/test-stderr.txt"] + verifier_stderr_path = local_path_from_uri( + verifier_stderr_ref.uri, + context="Harbor verifier stderr for unmeasurable trace", + ) + assert "trace" in verifier_stderr_path.read_text(encoding="utf-8").lower() + assert set(unmeasurable_trial.metrics) - {"reward"} == set() + unmeasurable_reward_ref = unmeasurable_trial.resources.get("log:verifier/reward.json") + if unmeasurable_reward_ref is not None: + unmeasurable_reward_path = local_path_from_uri( + unmeasurable_reward_ref.uri, + context="Harbor verifier reward for unmeasurable trace", + ) + unmeasurable_payload = json.loads(unmeasurable_reward_path.read_text(encoding="utf-8")) + assert set(unmeasurable_payload) == {"reward"} diff --git a/plugins/nemo-eval-author/tests/test_eval_author_run.py b/plugins/nemo-eval-author/tests/test_eval_author_run.py index 7963b06348..722af79f7c 100644 --- a/plugins/nemo-eval-author/tests/test_eval_author_run.py +++ b/plugins/nemo-eval-author/tests/test_eval_author_run.py @@ -1,14 +1,18 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +"""The orchestration boundary accepts only authoring inputs.""" + +import inspect from dataclasses import dataclass from pathlib import Path from typing import Any import pytest from nemo_eval_author_plugin.eval_author import run as eval_author_run +from nemo_eval_author_plugin.eval_author.agent import EvalAuthor from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig, EvalAuthorResult -from nemo_experimentalist_plugin.entities import Dataset, DatasetRef, Task +from nemo_experimentalist_plugin.entities import Dataset, DatasetRef, ResourceRef, Task from nemo_insights_plugin.entities import Insight @@ -21,71 +25,42 @@ async def close(self) -> None: self.closed = True -@dataclass -class BackendFactoryCall: - client: ClosingClient - experiments_output: str - - -@dataclass -class AgentCodeCall: - workspace: str - agent: str | Path - dest: Path - - -@dataclass -class EvalAuthorFactoryCall: - experiment_dir: Path - config: EvalAuthorConfig - - -@dataclass -class EvalAuthorCall: - insight: Insight - agent_path: Path - task_template: Task - train_dataset: Dataset - validation_dataset: Dataset - client: ClosingClient - - class FakeBackend: def __init__(self, insight: Insight) -> None: self.insight = insight - self.insight_calls: list[dict[str, str]] = [] - self.agent_code_calls: list[AgentCodeCall] = [] + self.insight_calls: list[tuple[str, str]] = [] + self.agent_calls: list[tuple[str, str | Path, Path]] = [] async def get_insight(self, *, workspace: str, insight_id: str) -> Insight: - self.insight_calls.append({"workspace": workspace, "insight_id": insight_id}) + self.insight_calls.append((workspace, insight_id)) return self.insight async def get_agent_code(self, *, workspace: str, agent: str | Path, dest: Path) -> None: - self.agent_code_calls.append(AgentCodeCall(workspace=workspace, agent=agent, dest=dest)) + self.agent_calls.append((workspace, agent, dest)) class FakeDatasetFactory: def __init__(self) -> None: - self.train = Dataset(id="train") - self.validation = Dataset(id="validation") - self.template = Task(id="template-task", uri="file:///template") - self.dataset_refs: list[tuple[str, DatasetRef]] = [] - self.template_refs: list[tuple[str, DatasetRef]] = [] - - def build_dataset(self, evaluator_type: str, dataset_ref: DatasetRef) -> Dataset: - self.dataset_refs.append((evaluator_type, dataset_ref)) - if dataset_ref.metadata.get("id") == "validation": - return self.validation - return self.train + self.template = Task(id="template", uri="file:///template") + self.template_calls: list[tuple[str, DatasetRef]] = [] + self.dataset_calls: list[tuple[str, DatasetRef]] = [] + self.datasets: list[Dataset] = [] def build_task_template(self, evaluator_type: str, template_ref: DatasetRef) -> Task: - self.template_refs.append((evaluator_type, template_ref)) + self.template_calls.append((evaluator_type, template_ref)) return self.template + def build_dataset(self, evaluator_type: str, dataset_ref: DatasetRef) -> Dataset: + self.dataset_calls.append((evaluator_type, dataset_ref)) + dataset = Dataset(id=Path(dataset_ref.uri).name, source=ResourceRef(uri=Path(dataset_ref.uri).as_uri())) + self.datasets.append(dataset) + return dataset + class FakeEvalAuthor: def __init__(self) -> None: - self.call: EvalAuthorCall | None = None + self.call: tuple[Insight, Path, Task, Dataset, Dataset, ClosingClient] | None = None + self.insight_suite = Dataset(id="insight-suite") async def run( self, @@ -97,23 +72,19 @@ async def run( *, client: ClosingClient, ) -> EvalAuthorResult: - self.call = EvalAuthorCall( - insight=insight, - agent_path=agent_path, - task_template=task_template, - train_dataset=train_dataset, - validation_dataset=validation_dataset, - client=client, - ) + self.call = (insight, agent_path, task_template, train_dataset, validation_dataset, client) return EvalAuthorResult( train_dataset=train_dataset, validation_dataset=validation_dataset, - summary="Eval Author complete", + insight_suite=self.insight_suite, + insight_suite_identity=f"sha256:{'a' * 64}", + metric_keys=("uses_correct_tool",), + summary="Eval Author complete.", ) @pytest.mark.asyncio -async def test_run_eval_author_builds_and_runs_complete_contract( +async def test_run_eval_author_resolves_inputs_and_returns_datasets( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: @@ -122,196 +93,149 @@ async def test_run_eval_author_builds_and_runs_complete_contract( workspace="workspace-a", title="failure", description="description", - agent=str(tmp_path / "agent-src"), + agent="insight-agent", trace_refs=["trace-1"], ) backend = FakeBackend(insight) dataset_factory = FakeDatasetFactory() eval_author = FakeEvalAuthor() - backend_calls: list[BackendFactoryCall] = [] - eval_author_calls: list[EvalAuthorFactoryCall] = [] - litellm_calls: list[bool] = [] - - def make_backend( - *, - client: ClosingClient, - experiments_output: str, - ) -> FakeBackend: - backend_calls.append(BackendFactoryCall(client=client, experiments_output=experiments_output)) - return backend - - def build_eval_author_agent(*, experiment_dir: Path, config: EvalAuthorConfig) -> FakeEvalAuthor: - eval_author_calls.append(EvalAuthorFactoryCall(experiment_dir=experiment_dir, config=config)) - return eval_author - - monkeypatch.setattr(eval_author_run, "make_client", lambda base_url: client) - monkeypatch.setattr(eval_author_run, "make_experimentalist_backend", make_backend) + monkeypatch.setattr(eval_author_run, "make_client", lambda _: client) + monkeypatch.setattr(eval_author_run, "make_experimentalist_backend", lambda **_: backend) monkeypatch.setattr(eval_author_run, "DatasetFactory", lambda: dataset_factory) - monkeypatch.setattr(eval_author_run, "build_eval_author_agent", build_eval_author_agent) - monkeypatch.setattr(eval_author_run, "_enable_litellm_drop_params", lambda: litellm_calls.append(True)) - - config = EvalAuthorConfig(max_traces=2) - train_ref = DatasetRef(uri=str(tmp_path / "train"), metadata={"id": "train"}) - validation_ref = DatasetRef(uri=str(tmp_path / "validation"), metadata={"id": "validation"}) - template_path = tmp_path / "template" - template_path.mkdir() - template_ref = DatasetRef(uri=str(template_path), metadata={"id": "task-template"}) + monkeypatch.setattr(eval_author_run, "build_eval_author_agent", lambda **_: eval_author) + monkeypatch.setattr(eval_author_run, "_enable_litellm_drop_params", lambda: None) + template = tmp_path / "template" + template.mkdir() + (template / "task.toml").write_text("template\n", encoding="utf-8") + train = tmp_path / "train" + validation = tmp_path / "validation" + train.mkdir() + validation.mkdir() result = await eval_author_run.run_eval_author( - insight="insight-remote-123", - train_dataset=train_ref, - validation_dataset=validation_ref, - task_template=template_ref, - experiment_dir=tmp_path / "eval_author", + insight="insight-123", + train_dataset=DatasetRef(uri=str(train)), + validation_dataset=DatasetRef(uri=str(validation)), + task_template=DatasetRef(uri=str(template)), + experiment_dir=tmp_path / "experiment", workspace="workspace-a", base_url="http://platform.test", - config=config, + config=EvalAuthorConfig(), ) - experiment_dir = (tmp_path / "eval_author").resolve() - assert result.summary == "Eval Author complete" - assert result.train_dataset is dataset_factory.train - assert result.validation_dataset is dataset_factory.validation - assert litellm_calls == [True] - assert backend_calls == [ - BackendFactoryCall(client=client, experiments_output=str(experiment_dir)), - ] - assert backend.insight_calls == [{"workspace": "workspace-a", "insight_id": "insight-remote-123"}] - assert backend.agent_code_calls == [ - AgentCodeCall( - workspace="workspace-a", - agent=insight.agent, - dest=experiment_dir / "eval_author" / "source-agent", - ) - ] - assert dataset_factory.dataset_refs == [("harbor", train_ref), ("harbor", validation_ref)] - assert dataset_factory.template_refs == [ - ( - "harbor", - template_ref.model_copy(update={"uri": str(experiment_dir / "dataset" / "task-template")}), - ) + experiment_dir = (tmp_path / "experiment").resolve() + assert result.train_dataset is dataset_factory.datasets[0] + assert result.validation_dataset is dataset_factory.datasets[1] + assert result.insight_suite is eval_author.insight_suite + assert result.insight_suite_identity == f"sha256:{'a' * 64}" + assert result.metric_keys == ("uses_correct_tool",) + assert backend.insight_calls == [("workspace-a", "insight-123")] + assert backend.agent_calls == [ + ("workspace-a", "insight-agent", experiment_dir / "eval_author" / "source-agent"), ] - assert eval_author_calls == [EvalAuthorFactoryCall(experiment_dir=experiment_dir, config=config)] - assert eval_author.call == EvalAuthorCall( - insight=insight, - agent_path=experiment_dir / "eval_author" / "source-agent", - task_template=dataset_factory.template, - train_dataset=dataset_factory.train, - validation_dataset=dataset_factory.validation, - client=client, + assert [call[0] for call in dataset_factory.dataset_calls] == ["harbor", "harbor"] + assert dataset_factory.template_calls[0][0] == "harbor" + assert eval_author.call == ( + insight, + experiment_dir / "eval_author" / "source-agent", + dataset_factory.template, + dataset_factory.datasets[0], + dataset_factory.datasets[1], + client, ) assert client.closed +def test_public_apis_accept_train_validation_and_generated_task_inputs() -> None: + orchestration = inspect.signature(eval_author_run.run_eval_author).parameters + agent_run = inspect.signature(EvalAuthor.run).parameters + agent_private_run = inspect.signature(EvalAuthor._run).parameters + + assert {"insight", "task_template", "train_dataset", "validation_dataset"} <= set(orchestration) + assert {"request", "reference_task_sets"}.isdisjoint(orchestration) + expected = { + "self", + "insight", + "agent_path", + "task_template", + "train_dataset", + "validation_dataset", + "client", + } + assert set(agent_run) == expected + assert set(agent_private_run) == expected + + @pytest.mark.asyncio async def test_run_eval_author_hydrates_fileset_task_template( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: - download_calls: list[dict[str, str]] = [] + downloads: list[tuple[str, str, str]] = [] class FakeFiles: async def download(self, *, remote_path: str, local_path: str, workspace: str) -> None: - download_calls.append({"remote_path": remote_path, "local_path": local_path, "workspace": workspace}) + downloads.append((remote_path, local_path, workspace)) destination = Path(local_path) destination.mkdir(parents=True) - (destination / "task.toml").write_text("template", encoding="utf-8") + (destination / "task.toml").write_text("template\n", encoding="utf-8") client = ClosingClient(files=FakeFiles()) - insight = Insight(workspace="workspace-a", title="failure", description="description", agent="insight-agent") - backend = FakeBackend(insight) + backend = FakeBackend( + Insight(workspace="workspace-a", title="failure", description="description", agent="insight-agent") + ) dataset_factory = FakeDatasetFactory() - eval_author = FakeEvalAuthor() - - monkeypatch.setattr(eval_author_run, "make_client", lambda base_url: client) + monkeypatch.setattr(eval_author_run, "make_client", lambda _: client) monkeypatch.setattr(eval_author_run, "make_experimentalist_backend", lambda **_: backend) monkeypatch.setattr(eval_author_run, "DatasetFactory", lambda: dataset_factory) - monkeypatch.setattr(eval_author_run, "build_eval_author_agent", lambda **_: eval_author) + monkeypatch.setattr(eval_author_run, "build_eval_author_agent", lambda **_: FakeEvalAuthor()) monkeypatch.setattr(eval_author_run, "_enable_litellm_drop_params", lambda: None) + template_ref = DatasetRef(uri="fileset://workspace-a/template") + train = tmp_path / "train" + validation = tmp_path / "validation" + train.mkdir() + validation.mkdir() - template_ref = DatasetRef(uri="fileset://workspace-a/task-template", metadata={"id": "task-template"}) - experiment_dir = (tmp_path / "eval_author").resolve() await eval_author_run.run_eval_author( - insight="insight-remote-123", - train_dataset=DatasetRef(uri="train", metadata={"id": "train"}), - validation_dataset=DatasetRef(uri="validation", metadata={"id": "validation"}), + insight="insight-123", + train_dataset=DatasetRef(uri=str(train)), + validation_dataset=DatasetRef(uri=str(validation)), task_template=template_ref, - experiment_dir=experiment_dir, + experiment_dir=tmp_path / "experiment", workspace="workspace-a", - base_url="http://platform.test", + base_url=None, config=EvalAuthorConfig(), ) - staged_path = experiment_dir / "dataset" / "task-template" - assert download_calls == [ - { - "remote_path": template_ref.uri, - "local_path": str(staged_path), - "workspace": "workspace-a", - } - ] - assert dataset_factory.template_refs == [("harbor", template_ref.model_copy(update={"uri": str(staged_path)}))] + staged = (tmp_path / "experiment").resolve() / "dataset" / "task-template" + assert downloads == [(template_ref.uri, str(staged), "workspace-a")] + assert dataset_factory.template_calls == [("harbor", template_ref.model_copy(update={"uri": str(staged)}))] + assert [Path(ref.uri).name for _, ref in dataset_factory.dataset_calls] == ["train", "validation"] assert client.closed @pytest.mark.asyncio -async def test_run_eval_author_uses_agent_override( +async def test_run_eval_author_closes_client_on_failure( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: client = ClosingClient() - insight = Insight(workspace="workspace-a", title="failure", description="description", agent="insight-agent") - backend = FakeBackend(insight) - dataset_factory = FakeDatasetFactory() - eval_author = FakeEvalAuthor() - - monkeypatch.setattr(eval_author_run, "make_client", lambda base_url: client) - monkeypatch.setattr(eval_author_run, "make_experimentalist_backend", lambda **_: backend) - monkeypatch.setattr(eval_author_run, "DatasetFactory", lambda: dataset_factory) - monkeypatch.setattr(eval_author_run, "build_eval_author_agent", lambda **_: eval_author) - monkeypatch.setattr(eval_author_run, "_enable_litellm_drop_params", lambda: None) - - override = tmp_path / "override-agent" - template = tmp_path / "template" - template.mkdir() - await eval_author_run.run_eval_author( - insight="insight-remote-123", - agent=override, - train_dataset=DatasetRef(uri="train", metadata={"id": "train"}), - validation_dataset=DatasetRef(uri="validation", metadata={"id": "validation"}), - task_template=DatasetRef(uri=str(template)), - experiment_dir=tmp_path / "eval_author", - workspace="workspace-a", - base_url="http://platform.test", - config=EvalAuthorConfig(), + monkeypatch.setattr(eval_author_run, "make_client", lambda _: client) + monkeypatch.setattr( + eval_author_run, + "make_experimentalist_backend", + lambda **_: (_ for _ in ()).throw(RuntimeError("backend failed")), ) - assert backend.agent_code_calls[0].agent == override - assert client.closed - - -@pytest.mark.asyncio -async def test_run_eval_author_closes_client_when_backend_creation_fails( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - client = ClosingClient() - - def fail_backend_creation(**_: object) -> object: - raise RuntimeError("backend creation failed") - - monkeypatch.setattr(eval_author_run, "make_client", lambda base_url: client) - monkeypatch.setattr(eval_author_run, "make_experimentalist_backend", fail_backend_creation) - - with pytest.raises(RuntimeError, match="backend creation failed"): + with pytest.raises(RuntimeError, match="backend failed"): await eval_author_run.run_eval_author( - insight="insight-remote-123", + insight="insight-123", train_dataset=DatasetRef(uri="train"), validation_dataset=DatasetRef(uri="validation"), task_template=DatasetRef(uri="template"), - experiment_dir=tmp_path / "eval_author", + experiment_dir=tmp_path, workspace="workspace-a", - base_url="http://platform.test", + base_url=None, config=EvalAuthorConfig(), ) diff --git a/plugins/nemo-eval-author/tests/test_plugin_boundary.py b/plugins/nemo-eval-author/tests/test_plugin_boundary.py index bc4cc0e20f..50c07a70b1 100644 --- a/plugins/nemo-eval-author/tests/test_plugin_boundary.py +++ b/plugins/nemo-eval-author/tests/test_plugin_boundary.py @@ -31,7 +31,7 @@ # # client -> make_client, the platform client factory # ...components -> the cache module, for run artifacts -# ...dataset_staging -> stage_task_template +# ...dataset_staging -> stage_eval_author_inputs # ...evaluator.base -> EvaluatorType # ...evaluator.factory -> DatasetFactory # ...evaluator.harbor -> HarborDataset diff --git a/plugins/nemo-experimentalist/README.md b/plugins/nemo-experimentalist/README.md index f1f4e00e35..ad2bfd610d 100644 --- a/plugins/nemo-experimentalist/README.md +++ b/plugins/nemo-experimentalist/README.md @@ -181,6 +181,40 @@ The `--config` YAML is the other kind of configuration: it holds what *one exper does (`max_rounds`, `max_survivors`, per-component tuning) and takes no environment override, so the file is an accurate record of the run. +### Objective function and regression metrics + +Declare what the optimizer should improve separately from what it must preserve. +`objective_function` is one ordered list: each item may be a raw evaluator +metric or an aggregate metric produced by the evaluator. The optimizer only receives +reported metric values and the declared policy; it does not evaluate expressions, +invent weights, or encode a selection algorithm. + +A single evaluator-produced aggregate metric: + +```yaml +objective_function: + - name: quality + direction: maximize +``` + +Several metrics, for example lower token use and cost, with a guardrail: + +```yaml +objective_function: + - name: tokens + direction: minimize + - name: cost + direction: minimize +regression_metrics: + - name: success_rate + direction: maximize +``` + +For an insight-driven run, Eval Author's authored insight metrics replace the +run-level objective metrics. The configured objective targets move to +`regression_metrics`, alongside the existing guardrails, so the insight is +improved without giving up the run's original priorities. + The agent under test is separate: it reads `AUT_MODEL_NAME` plus `OPENAI_API_KEY` / `OPENAI_BASE_URL`, which are the only variables forwarded into the evaluation container. diff --git a/plugins/nemo-experimentalist/examples/tau3-nooa-agent/.env.example b/plugins/nemo-experimentalist/examples/tau3-nooa-agent/.env.example index 036ce78731..33ae2ebbda 100644 --- a/plugins/nemo-experimentalist/examples/tau3-nooa-agent/.env.example +++ b/plugins/nemo-experimentalist/examples/tau3-nooa-agent/.env.example @@ -13,7 +13,7 @@ export NEMO_EXPERIMENTALIST_API_BASE="$INFERENCE_API_BASE" # Upstream Tau3 currently reads these compatibility variable names. export TAU2_USER_MODEL=openai/openai/openai/gpt-5.6-luna export TAU2_NL_ASSERTIONS_MODEL=openai/openai/openai/gpt-5.6-luna -export AUT_MODEL_NAME=openai/openai/openai/gpt-5.6-luna +export AUT_MODEL_NAME=openai/openai/openai/gpt-5-mini export NEMO_EXPERIMENTALIST_MODELS_SMART=openai/openai/openai/gpt-5.6-sol export NEMO_EXPERIMENTALIST_MODELS_MID=openai/openai/openai/gpt-5.6-terra diff --git a/plugins/nemo-experimentalist/examples/tau3-nooa-agent/experimentalist-smoke.yaml b/plugins/nemo-experimentalist/examples/tau3-nooa-agent/experimentalist-smoke.yaml index aca2abd9de..c4b00ce3fc 100644 --- a/plugins/nemo-experimentalist/examples/tau3-nooa-agent/experimentalist-smoke.yaml +++ b/plugins/nemo-experimentalist/examples/tau3-nooa-agent/experimentalist-smoke.yaml @@ -6,7 +6,6 @@ min_rounds_before_stopping: 1 max_survivors: 1 max_candidates: 1 max_trajectory_tasks: 2 -max_train_batch_tasks: 4 train_batch_seed: 20260727 disable_trajectory_scoring: true disable_convergence_check: true diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/config.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/config.py index 1517ecd351..2df38fc1a3 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/config.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/config.py @@ -20,7 +20,7 @@ """ from pathlib import Path -from typing import Any +from typing import Any, Literal, Self from nemo_eval_author_plugin.eval_author.models import EvalAuthorConfig from nemo_experimentalist_plugin.experimentalist.components.analyzer import AnalyzerConfig @@ -51,6 +51,35 @@ class CandidateStorageConfig(BaseModel): pr_labels: list[str] = Field(default_factory=list) +class MetricTarget(BaseModel): + """One evaluator-produced metric and the desired direction of change.""" + + name: str = Field(min_length=1, description="Exact metric name emitted by the evaluator.") + direction: Literal["maximize", "minimize"] = Field( + description="Whether higher or lower values are better for this target." + ) + + +def pareto_objectives(metrics: dict[str, float], objective_function: list[MetricTarget]) -> dict[str, float]: + """Project evaluator metrics onto the configured objectives for Pareto ranking. + + The generic Pareto utility maximizes every dimension. Minimized objective + values are sign-inverted here; regression metrics are intentionally absent. + """ + objectives: dict[str, float] = {} + for target in objective_function: + value = metrics.get(target.name) + if value is None: + return {} + objectives[target.name] = float(value) if target.direction == "maximize" else -float(value) + return objectives + + +def has_metric_dimensions(metrics: dict[str, float], targets: list[MetricTarget]) -> bool: + """Return whether an evaluator result contains every required metric target.""" + return all(target.name in metrics for target in targets) + + class EvolutionaryOptimizerConfig(BaseModel): """Parameters for one optimizer run, read from ``--config`` and nothing else. @@ -97,6 +126,15 @@ def reject_legacy_curator_config(cls, data: Any) -> Any: disable_convergence_check: bool = Field( default=False, description="Stop only on max_rounds, never on the terminator's convergence judgement." ) + objective_function: list[MetricTarget] = Field( + default_factory=lambda: [MetricTarget(name="reward", direction="maximize")], + min_length=1, + description="Ordered evaluator metric targets this run should improve.", + ) + regression_metrics: list[MetricTarget] = Field( + default_factory=list, + description="Metric target(s) that must not regress while the objective improves.", + ) source: AgentSourceConfig = Field(default_factory=AgentSourceConfig) storage: CandidateStorageConfig = Field(default_factory=CandidateStorageConfig) goal_config: GoalTreeConfig = Field(default_factory=GoalTreeConfig) @@ -105,3 +143,32 @@ def reject_legacy_curator_config(cls, data: Any) -> Any: proposer: ProposerConfig = Field(default_factory=ProposerConfig) evaluator: dict[str, Any] = Field(default_factory=dict) eval_author: EvalAuthorConfig = Field(default_factory=EvalAuthorConfig) + + @model_validator(mode="after") + def validate_metric_contract(self) -> Self: + objective_names = [target.name for target in self.objective_function] + if len(objective_names) != len(set(objective_names)): + raise ValueError("objective_function target names must be unique") + regression_names = [target.name for target in self.regression_metrics] + if len(regression_names) != len(set(regression_names)): + raise ValueError("regression_metrics target names must be unique") + overlap = set(objective_names).intersection(regression_names) + if overlap: + raise ValueError( + "A metric cannot be both an objective and a regression target: " + ", ".join(sorted(overlap)) + ) + return self + + def optimization_policy(self) -> str: + """Render the declared metric contract for optimizer-facing reasoning prompts.""" + + def render(target: MetricTarget) -> str: + return f"{target.name} ({target.direction})" + + objectives = ", ".join(render(target) for target in self.objective_function) + regressions = ", ".join(render(target) for target in self.regression_metrics) or "none" + return ( + f"Optimize these objective metric(s): {objectives}. " + f"Do not regress these metric(s): {regressions}. " + "Metric values, including aggregates, are produced by the evaluator; do not invent formulas or weights." + ) diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/entities.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/entities.py index 45c3cb3e26..f3f5075878 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/entities.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/entities.py @@ -259,6 +259,15 @@ def list_tasks(self) -> Sequence[Task]: """ return list(self.tasks) + def add_tasks(self, tasks: list[Task]) -> None: + """Add tasks to this dataset's durable backing store. + + Dataset implementations define how task artifacts are imported. Callers + must use this method instead of mutating ``tasks`` directly so an + evaluator can preserve the task files needed at evaluation time. + """ + raise NotImplementedError(f"{type(self).__name__} does not support adding tasks") + async def validate(self) -> None: """Validate authored dataset content without running evaluation trials. @@ -489,10 +498,9 @@ class Candidate(NemoEntity, entity_type="candidate"): rewards: dict[str, RewardRecord] = Field( default_factory=dict, description=( - "Measurements keyed by reward channel. An open set: 'train', 'validation', " - "'insight' and 'validation-trajectory' today. A channel is a measurement, not a " - "dataset split — trajectory scoring is a second measurement of the validation " - "split — so adding one costs no entity change." + "Measurements keyed by reward channel. This is an open set that includes " + "'train', 'validation', and 'validation-trajectory'. A channel is a measurement, " + "not a dataset split, so adding one costs no entity change." ), ) trajectory_detail: dict[str, Any] | None = Field( diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/analyzer.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/analyzer.py index 9797b09e08..9728abe8a2 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/analyzer.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/analyzer.py @@ -34,6 +34,8 @@ from .tools import GuardedShellTools from .util import load_framework_skills +logger = logging.getLogger(__name__) + class AnalyzerConfig(BaseModel): """Configure tuning parameters for AgentAnalyzer.""" @@ -170,6 +172,7 @@ class TrialAnalysis(BaseModel): task_id: str trial_id: str + selection_reason: str metrics: dict[str, float] diagnostic: Diagnostic @@ -182,6 +185,7 @@ def __repr__(self) -> str: """ metrics = ", ".join(f"{name}: {value:.3f}" for name, value in self.metrics.items()) or "no metrics" lines = [f"### {self.task_id} / {self.trial_id} ({metrics})"] + lines.append(f"Selected for analysis: {self.selection_reason}") lines.append(f"Outcome: {self.diagnostic.outcome}") lines.append(f"Summary: {self.diagnostic.summary}") if self.diagnostic.failure_point is not None: @@ -190,6 +194,13 @@ def __repr__(self) -> str: return "\n".join(lines) +class TrialSelection(BaseModel): + """One trial selected for analysis, with the reason it merits attention.""" + + trial_id: str = Field(description="ID of a trial from the evaluation result.") + reason: str = Field(description="Evidence-based reason this trial should be analyzed.") + + class AgentAnalysis(BaseModel): """Represent full analysis output for one agent: trials, failure classes, and peer comparisons.""" @@ -261,16 +272,21 @@ async def select_trials( agent_id: str, dataset: Dataset, evaluation: EvaluationResult, - ) -> Sequence[TrialResult]: - """Pick which trials to analyze in depth. Return their TrialResult objects. + objective_metrics: list[dict[str, str]], + regression_metrics: list[dict[str, str]], + ) -> list[TrialSelection]: + """Pick which trials to analyze in depth and explain each choice. Args: agent_id: The agent to analyze. dataset: The dataset to analyze. evaluation: The evaluation result to analyze. + objective_metrics: Metrics the optimization run should improve. + regression_metrics: Metrics that must not worsen. Returns: - Sequence[TrialResult]: The selected trials. + list[TrialSelection]: Selected evaluation trial IDs and the evidence-based + reason each was selected. ## Step 1: Get task and trial objects @@ -290,21 +306,39 @@ async def select_trials( ## Step 3: Triage — pick up to {self._config.max_trials} trials + Objective metrics: {objective_metrics} + Regression metrics: {regression_metrics} + Prefer trials where: + - an objective metric is low relative to the other trials or is the + clearest evidence of why that objective is not improving - status/error indicates the evaluator did not complete cleanly - - one or more numeric metric values are below the task's expected passing - value - the trace reference is missing or unloadable - outputs/resources suggest repeated failures across task ids + Regression metrics are guardrails. Do not select a trial solely because + a regression metric is low unless it exposes why an objective-focused + change would violate that guardrail. + Metric names are evaluator-defined. Do not assume particular metric names, result directories, private checks, or split paths. - ## Step 4: Return TrialResult objects + ## Step 4: Return selected trial IDs and a reason for each ```python - return selected_trials + return [ + TrialSelection( + trial_id=trial.id, + reason="Concise evidence-based reason to inspect this trace.", + ) + for trial in selected_trials + ] ``` + + Every ``trial_id`` must identify a trial from ``evaluation.trials``. + State the concrete objective shortfall that makes each selected trial + useful. Mention a regression risk only as a constraint on that + objective-focused analysis. """ ... @@ -317,6 +351,8 @@ async def classify_failures( agent_id: str, diagnoses: list[Diagnostic], trials: Sequence[TrialResult], + objective_metrics: list[dict[str, str]], + regression_metrics: list[dict[str, str]], ) -> FailureClassification: """Classify diagnoses into systematic vs. one-off and agent vs. mechanical failures. @@ -333,6 +369,12 @@ async def classify_failures( an agent logic error (optimizable) or a mechanical error (needs an infra fix). Do not penalise the agent for mechanical errors. + Center systematic failures and root causes on `objective_metrics`: explain + how the trace behavior causes an objective metric to underperform. + `regression_metrics` are guardrails; report their risks separately and do + not classify a regression-only shortfall as the primary failure pattern. + Do not invent formulas or weights for evaluator metrics. + Return a FailureClassification with: - `systematic`: list of SystematicFailure (root_cause, affected_tasks, pattern) - `mechanical`: list of MechanicalError (task_id, issue_type, description) @@ -346,6 +388,8 @@ async def compare_with_peers( evaluation: EvaluationResult, diagnoses: list[Diagnostic], peer_evaluations: dict[str, EvaluationResult] | None = None, + objective_metrics: list[dict[str, str]] | None = None, + regression_metrics: list[dict[str, str]] | None = None, ) -> PeerComparison: """Compare this agent to peers and return divergent trials and complementary patterns. @@ -366,17 +410,31 @@ async def compare_with_peers( agent_id, evaluation, peer_evaluations, + metric_directions=self._metric_directions(objective_metrics or [], regression_metrics or []), k=self._config.max_divergent_pairs, ) - complementary_raw = self._find_complementary_failures(agent_id, evaluation, peer_evaluations) + complementary_raw = self._find_complementary_failures( + agent_id, + evaluation, + peer_evaluations, + metric_directions=self._metric_directions(objective_metrics or [], regression_metrics or []), + ) - return await self._narrate_peer_comparison(agent_id, top_divergent, complementary_raw, diagnoses) + return await self._narrate_peer_comparison( + agent_id, + top_divergent, + complementary_raw, + diagnoses, + objective_metrics or [], + regression_metrics or [], + ) def _select_divergent_pairs( self, agent_id: str, evaluation: EvaluationResult, peer_evaluations: dict[str, EvaluationResult], + metric_directions: dict[str, str], k: int = 3, ) -> list[dict[str, Any]]: """Pick top-k divergent (peer, task) pairs ordered by absolute score delta. @@ -404,7 +462,11 @@ def _select_divergent_pairs( for task_id in sorted(set(focal_means) & set(peer_means)): fm = focal_means[task_id] pm = peer_means[task_id] - focal_delta = {m: fm.get(m, 0.0) - pm.get(m, 0.0) for m in sorted(set(fm) | set(pm))} + focal_delta = { + metric: (fm.get(metric, 0.0) - pm.get(metric, 0.0)) + * (1.0 if metric_directions.get(metric, "maximize") == "maximize" else -1.0) + for metric in sorted(set(fm) | set(pm)) + } magnitude = sum(abs(v) for v in focal_delta.values()) if magnitude == 0.0: continue @@ -449,11 +511,19 @@ def _task_metric_means(self, evaluation: EvaluationResult) -> dict[str, dict[str for task_id, metrics in per_task.items() } + @staticmethod + def _metric_directions( + objective_metrics: list[dict[str, str]], regression_metrics: list[dict[str, str]] + ) -> dict[str, str]: + """Return metric directions, defaulting dimensions outside the contract to maximize.""" + return {metric["name"]: metric["direction"] for metric in [*objective_metrics, *regression_metrics]} + def _find_complementary_failures( self, agent_id: str, evaluation: EvaluationResult, peer_evaluations: dict[str, EvaluationResult], + metric_directions: dict[str, str], ) -> dict[str, dict[str, dict[str, list[str]]]]: """Find tasks where agents split on leaders vs. trailers, per metric. @@ -482,10 +552,14 @@ def _find_complementary_failures( } if len(values) < 2 or min(values.values()) == max(values.values()): continue - best = max(values.values()) + best = ( + max(values.values()) + if metric_directions.get(metric, "maximize") == "maximize" + else min(values.values()) + ) per_metric[metric] = { "leaders": sorted(agent for agent, value in values.items() if value == best), - "trailers": sorted(agent for agent, value in values.items() if value < best), + "trailers": sorted(agent for agent, value in values.items() if value != best), } if per_metric: complementary[task_id] = per_metric @@ -498,6 +572,8 @@ async def _narrate_peer_comparison( top_divergent: list[dict[str, Any]], complementary_raw: dict[str, dict[str, dict[str, list[str]]]], diagnoses: list[Diagnostic], + objective_metrics: list[dict[str, str]], + regression_metrics: list[dict[str, str]], ) -> PeerComparison: """Write the DivergentTrial and ComplementaryPattern narratives from pre-computed data. @@ -516,6 +592,11 @@ async def _narrate_peer_comparison( `{task_id: {metric: {"leaders": [agents], "trailers": [agents]}}}` — per-metric splits between the best-scoring agents and the rest. + `objective_metrics` identifies metrics to improve and + `regression_metrics` identifies metrics to preserve. Use them to explain + whether a divergence is desirable or a regression risk; do not create a + scalar ranking. + ## For each divergent pair Produce one DivergentTrial: @@ -571,6 +652,8 @@ async def run( client: AsyncNeMoPlatform | None = None, nmp_workspace: str | None = None, agent_spec: Path | None = None, + objective_metrics: list[dict[str, str]] | None = None, + regression_metrics: list[dict[str, str]] | None = None, ) -> AgentAnalysis: """Run the full analysis pipeline for one agent in one optimization round. @@ -585,6 +668,8 @@ async def run( nmp_workspace: NeMo Platform (Intake) workspace *name* — the request context for ``intake://`` trace lookups. Distinct from the constructor's ``workspace: Path`` (the filesystem eval dir). + objective_metrics: Active metrics to improve. + regression_metrics: Active metrics to preserve. Returns: AgentAnalysis: per-trial diagnostics, failure classification, and peer @@ -598,17 +683,35 @@ async def run( # trace-starved. Keying on availability prevents such a degraded result # from being replayed on a later run that *can* load those traces. intake_key = ":intake:1" if client is not None and nmp_workspace is not None else ":intake:0" - cache_key = cache.agent_hash(f"{agent_id}:evaluation:{evaluation.id}{round_key}{intake_key}") + objective_metrics = objective_metrics or [] + regression_metrics = regression_metrics or [] + cache_key = cache.agent_hash( + f"{agent_id}:evaluation:{evaluation.id}{round_key}{intake_key}:" + f"objective-metrics:{objective_metrics}:regression-metrics:{regression_metrics}" + ) cached = cache.load(self._workspace_path, cache_key, AgentAnalysis) if cached is not None: return cached - trials = await self.select_trials(agent_id, dataset, evaluation) + selections = await self.select_trials(agent_id, dataset, evaluation, objective_metrics, regression_metrics) + trials_by_id = {trial.id: trial for trial in evaluation.trials} + selected_trials: list[tuple[TrialResult, str]] = [] + for selection in selections: + trial = trials_by_id.get(selection.trial_id) + if trial is None: + logger.warning( + "Ignoring selected trial %s for %s because it is absent from evaluation %s", + selection.trial_id, + agent_id, + evaluation.id, + ) + continue + selected_trials.append((trial, selection.reason)) tasks_by_id = self._tasks_by_id(dataset) missing_task_diagnostics: dict[str, Diagnostic] = {} - trial_tasks: list[tuple[TrialResult, Task]] = [] - for trial in trials: + trial_tasks: list[tuple[TrialResult, Task, str]] = [] + for trial, selection_reason in selected_trials: task = tasks_by_id.get(trial.task_id) if task is None: missing_task_diagnostics[trial.id] = Diagnostic( @@ -618,9 +721,9 @@ async def run( root_cause="evaluation_result_references_unknown_task", ) continue - trial_tasks.append((trial, task)) + trial_tasks.append((trial, task, selection_reason)) - unique_tasks = {task.id: task for _, task in trial_tasks} + unique_tasks = {task.id: task for _, task, _ in trial_tasks} rationales_list = await asyncio.gather( *[ Rationalizer( @@ -652,15 +755,18 @@ async def run( task=task, agent_path=agent_path, rationale=rationales.get(task.id), + selection_reason=selection_reason, + objective_metrics=objective_metrics, + regression_metrics=regression_metrics, client=client, workspace=nmp_workspace, ) - for trial, task in trial_tasks + for trial, task, selection_reason in trial_tasks ], return_exceptions=True, ) diagnostics_by_trial_id = dict(missing_task_diagnostics) - for (trial, _), result in zip(trial_tasks, diagnoses_list, strict=True): + for (trial, _, _), result in zip(trial_tasks, diagnoses_list, strict=True): if isinstance(result, asyncio.CancelledError): raise result if isinstance(result, BaseException): @@ -683,17 +789,31 @@ async def run( TrialAnalysis( task_id=trial.task_id, trial_id=trial.id, + selection_reason=selection_reason, metrics={name: float(metric.value) for name, metric in trial.metrics.items()}, diagnostic=diagnostics_by_trial_id[trial.id], ) - for trial in trials + for trial, selection_reason in selected_trials if trial.id in diagnostics_by_trial_id ] diagnoses = [analysis.diagnostic for analysis in trial_analyses] classification, comparison = await asyncio.gather( - self.classify_failures(agent_id, diagnoses, trials), - self.compare_with_peers(agent_id, evaluation, diagnoses, peer_evaluations), + self.classify_failures( + agent_id, + diagnoses, + [trial for trial, _ in selected_trials], + objective_metrics, + regression_metrics, + ), + self.compare_with_peers( + agent_id, + evaluation, + diagnoses, + peer_evaluations, + objective_metrics, + regression_metrics, + ), ) analysis_out = AgentAnalysis( diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/cache.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/cache.py index bcb7a16e31..d62f41e176 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/cache.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/cache.py @@ -63,6 +63,17 @@ def trace_hash(trace_path: str | Path) -> str: return f"trace-{digest}" +def trace_uri_hash(trace_key: str) -> str: + """Return a namespaced SHA-256 digest for a trace cache identity. + + ``trace_key`` may include a trace-content digest or external trace URI plus + analysis inputs such as the metric contract. Unlike ``trace_hash``, it does + not read a filesystem path. + """ + digest = hashlib.sha256(trace_key.encode()).hexdigest() + return f"trace-uri-{digest}" + + def _cache_path(workspace: Path, key: str) -> Path: return workspace / "eval-and-optimize" / "cache" / f"{key}.json" diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/dataset_staging.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/dataset_staging.py index 8ce5f1e5e3..57a58988b1 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/dataset_staging.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/dataset_staging.py @@ -8,7 +8,7 @@ from pathlib import Path from urllib.parse import urlparse -from nemo_experimentalist_plugin.entities import DatasetRef, local_path_from_uri +from nemo_experimentalist_plugin.entities import Dataset, DatasetRef, local_path_from_uri from nemo_platform import AsyncNeMoPlatform @@ -21,6 +21,24 @@ class _StagedEvalAuthorInputs: task_template: DatasetRef +def distribute_insight_suite_tasks( + insight_suite: Dataset, + train_dataset: Dataset, + validation_dataset: Dataset, +) -> None: + """Assign Insight-suite tasks to validation/train at a deterministic 30/70 split. + + Eval Author retains the materialized suite as its provenance artifact. The + optimizer consumes its tasks through the train and validation datasets, + reserving the first 30 percent for validation and using the remaining 70 + percent for training feedback. + """ + tasks = list(insight_suite.list_tasks()) + validation_count = (3 * len(tasks) + 9) // 10 + validation_dataset.add_tasks(tasks[:validation_count]) + train_dataset.add_tasks(tasks[validation_count:]) + + def _local_directory(ref: DatasetRef) -> Path: path = local_path_from_uri(ref.uri, context="Eval Author input").resolve() if not path.is_dir(): diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/harbor.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/harbor.py index 5a5bc3d4ec..bdbd1244fd 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/harbor.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/evaluator/harbor.py @@ -1204,6 +1204,54 @@ def get_task(self, task_id: str) -> Task: return task raise ValueError(f"Task id not found in Harbor dataset {self.id!r}: {task_id}") + def add_tasks(self, tasks: list[Task]) -> None: + """Copy Harbor task directories into this dataset and register them. + + The destination dataset owns independent copies, so an Insight-suite + task can be evaluated through train or validation without depending on + the suite's original directory. + """ + if not tasks: + return + if self.source is None: + raise ValueError(f"Harbor dataset {self.id!r} has no source directory") + destination_root = local_path_from_uri(self.source.uri, context="Harbor dataset source").resolve() + if not destination_root.is_dir(): + raise ValueError(f"Harbor dataset source is not a directory: {destination_root}") + + imported: dict[str, Task] = {} + for task in tasks: + if task.uri is None: + raise ValueError(f"Harbor task {task.id!r} has no source directory") + source_dir = local_path_from_uri(task.uri, context=f"Harbor task {task.id!r}").resolve() + if not source_dir.is_dir() or not (source_dir / _HARBOR_CONFIG_FILENAME["name"]).is_file(): + raise ValueError(f"Harbor task {task.id!r} is not a task directory: {source_dir}") + destination = destination_root / task.id + staging = destination_root / f".{task.id}.staging-{uuid4().hex}" + backup = destination_root / f".{task.id}.backup-{uuid4().hex}" + shutil.copytree(source_dir, staging) + try: + self._from_task_dir(staging) + if destination.exists(): + logger.warning("Replacing existing Harbor task %s in dataset %s", task.id, self.id) + destination.rename(backup) + staging.rename(destination) + imported[task.id] = self._from_task_dir(destination) + except BaseException: + if destination.exists() and backup.exists(): + shutil.rmtree(destination) + if staging.exists(): + shutil.rmtree(staging) + if backup.exists() and not destination.exists(): + backup.rename(destination) + raise + else: + if backup.exists(): + shutil.rmtree(backup) + + self.tasks = [imported.pop(task.id, task) for task in self.tasks] + self.tasks.extend(imported.values()) + async def validate(self) -> None: """Validate selected task verifier syntax without executing verifier code.""" failures: list[HarborVerifierValidationFailure] = [] diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/insight_promotion.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/insight_promotion.py deleted file mode 100644 index ebb130f883..0000000000 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/insight_promotion.py +++ /dev/null @@ -1,524 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Rank Insight-suite tasks for possible manual promotion into validation.""" - -from __future__ import annotations - -import math -from collections.abc import Sequence -from dataclasses import dataclass, replace -from pathlib import Path - -from nemo_experimentalist_plugin.entities import ( - Candidate, - Dataset, - EvaluationResult, - Task, - TrialResult, - local_path_from_uri, -) - - -def candidate_suite_identity(candidate: Candidate) -> str | None: - """The Insight-suite identity this candidate's insight reward was measured against.""" - value = candidate.reward("insight").metadata.get("suite_identity") - return value if isinstance(value, str) else None - - -def candidate_metric_keys(candidate: Candidate) -> list[str]: - """The validated metric keys recorded alongside this candidate's insight reward. - - Anything that is not a list of strings reads as "not recorded" rather than being - coerced. This value gates whether a cached insight reward is reused, so coercing - (``[1]`` to ``["1"]``) would let malformed metadata pass as a valid measurement and - skip a fresh evaluation. - """ - value = candidate.reward("insight").metadata.get("metric_keys") - if not isinstance(value, list) or not all(isinstance(key, str) for key in value): - return [] - return list(value) - - -_GENERIC_METRIC_NAMES = frozenset({"reward", "score"}) -_MAX_REPEAT_SPREAD = 0.1 -_MIN_DISCRIMINATION = 1e-9 -_REPORT_SECTION_START = "" -_REPORT_SECTION_END = "" -_COMPARISON_SECTION_START = "" -_COMPARISON_SECTION_END = "" - - -@dataclass(frozen=True, slots=True) -class InsightSuiteProvenance: - """Runtime identities and local location for one finalized Insight suite.""" - - identity: str - scorer_identity: str - suite_path: Path - task_hashes: dict[str, dict[str, str]] - - -@dataclass(frozen=True, slots=True) -class InsightPromotionSuggestion: - """Evidence-backed recommendation to review one Insight-suite task.""" - - task_id: str - task_path: str - suite_identity: str - task_content_hash: str - verifier_hash: str - metric_name: str - discrimination: float - baseline_score: float - winner_score: float - completed_attempts: int - total_attempts: int - repeat_spread: float - candidate_count: int - diversity_score: float | None = None - - -@dataclass(frozen=True, slots=True) -class _TaskEvidence: - suggestion: InsightPromotionSuggestion - profile: dict[tuple[str, str], float] - - -def insight_suite_provenance(dataset: Dataset) -> InsightSuiteProvenance: - """Return validated content provenance carried by a finalized suite dataset.""" - identity = dataset.metadata.get("insight_suite_identity") - scorer_identity = dataset.metadata.get("insight_suite_scorer_identity") - raw_task_hashes = dataset.metadata.get("insight_suite_task_hashes") - if not isinstance(identity, str) or not identity.startswith("sha256:"): - raise ValueError("Finalized Insight suite is missing its content identity") - if not isinstance(scorer_identity, str) or not scorer_identity.startswith("sha256:"): - raise ValueError("Finalized Insight suite is missing its scorer identity") - if dataset.source is None: - raise ValueError("Finalized Insight suite is missing its local source path") - suite_path = local_path_from_uri(dataset.source.uri, context="Finalized Insight suite").resolve() - if not isinstance(raw_task_hashes, dict): - raise ValueError("Finalized Insight suite is missing task and verifier hashes") - task_hashes: dict[str, dict[str, str]] = {} - for task_id, raw_hashes in raw_task_hashes.items(): - if not isinstance(task_id, str) or not isinstance(raw_hashes, dict): - raise ValueError("Finalized Insight suite has invalid task hash provenance") - content_hash = raw_hashes.get("content_hash") - verifier_hash = raw_hashes.get("verifier_hash") - if not isinstance(content_hash, str) or not isinstance(verifier_hash, str): - raise ValueError(f"Finalized Insight task {task_id!r} has invalid content hashes") - task_hashes[task_id] = { - "content_hash": content_hash, - "verifier_hash": verifier_hash, - } - return InsightSuiteProvenance( - identity=identity, - scorer_identity=scorer_identity, - suite_path=suite_path, - task_hashes=task_hashes, - ) - - -def _validated_metric_value(value: float | int, *, context: str) -> float: - metric_value = float(value) - if not math.isfinite(metric_value) or not 0.0 <= metric_value <= 1.0: - raise ValueError(f"{context} must be finite and within [0, 1], got {value!r}") - return metric_value - - -def validate_insight_evaluation_result( - result: EvaluationResult, - *, - expected_metric_keys: Sequence[str] | None = None, -) -> tuple[str, ...]: - """Validate metrics before they become adaptive analysis or promotion evidence.""" - aggregate_keys = set(result.aggregate_metrics) - if not aggregate_keys: - raise ValueError("Insight evaluation produced no aggregate metrics") - if not aggregate_keys - _GENERIC_METRIC_NAMES: - raise ValueError("Insight evaluation produced no Insight-specific metric") - if expected_metric_keys is not None and aggregate_keys != set(expected_metric_keys): - raise ValueError( - "Insight evaluation aggregate metric keys are inconsistent: " - f"expected {sorted(expected_metric_keys)}, got {sorted(aggregate_keys)}" - ) - for metric_name, value in result.aggregate_metrics.items(): - _validated_metric_value(value, context=f"Insight aggregate metric {metric_name!r}") - - completed = [trial for trial in result.trials if trial.status == "completed"] - if not completed: - raise ValueError("Insight evaluation produced no completed trial evidence") - for trial in completed: - trial_keys = set(trial.metrics) - if trial_keys != aggregate_keys: - raise ValueError( - f"Insight trial {trial.id!r} metric keys are inconsistent: " - f"expected {sorted(aggregate_keys)}, got {sorted(trial_keys)}" - ) - for metric_name, metric in trial.metrics.items(): - _validated_metric_value( - metric.value, - context=f"Insight trial {trial.id!r} metric {metric_name!r}", - ) - return tuple(sorted(aggregate_keys)) - - -def stamp_insight_evaluation_result( - result: EvaluationResult, - provenance: InsightSuiteProvenance, -) -> EvaluationResult: - """Attach suite identity to aggregate and per-trial evidence.""" - suite_metadata = { - "insight_suite_identity": provenance.identity, - "insight_suite_scorer_identity": provenance.scorer_identity, - } - return result.model_copy( - update={ - "metadata": {**result.metadata, **suite_metadata}, - "trials": [ - trial.model_copy(update={"metadata": {**trial.metadata, **suite_metadata}}) for trial in result.trials - ], - } - ) - - -def _task_metric_values( - trials: Sequence[TrialResult], - *, - required_metrics: set[str], -) -> dict[str, list[float]] | None: - if len(trials) < 2 or any(trial.status != "completed" for trial in trials): - return None - values: dict[str, list[float]] = {metric_name: [] for metric_name in required_metrics} - for trial in trials: - if set(trial.metrics) != required_metrics: - return None - for metric_name, metric in trial.metrics.items(): - try: - value = _validated_metric_value( - metric.value, - context=f"Insight trial {trial.id!r} metric {metric_name!r}", - ) - except ValueError: - return None - values[metric_name].append(value) - return values - - -def _task_evidence( - task: Task, - candidates: Sequence[Candidate], - *, - baseline: Candidate, - winner: Candidate, - provenance: InsightSuiteProvenance, -) -> _TaskEvidence | None: - suite_candidates = [ - candidate for candidate in candidates if candidate_suite_identity(candidate) == provenance.identity - ] - metric_key_sets = {tuple(sorted(candidate_metric_keys(candidate))) for candidate in suite_candidates} - if len(metric_key_sets) != 1: - return None - required_metrics = set(next(iter(metric_key_sets), ())) - insight_metrics = required_metrics - _GENERIC_METRIC_NAMES - if not insight_metrics: - return None - - trials_by_candidate = { - candidate.label: [trial for trial in candidate.reward("insight").trials or () if trial.task_id == task.id] - for candidate in suite_candidates - } - values_by_candidate: dict[str, dict[str, list[float]]] = {} - for label, trials in trials_by_candidate.items(): - values = _task_metric_values(trials, required_metrics=required_metrics) - if values is None: - return None - values_by_candidate[label] = values - - if baseline.label not in values_by_candidate or winner.label not in values_by_candidate: - return None - total_attempts = sum(len(trials) for trials in trials_by_candidate.values()) - if not total_attempts: - return None - - profile: dict[tuple[str, str], float] = {} - repeat_spread = 0.0 - metric_improvements: dict[str, tuple[float, float, float]] = {} - for metric_name in sorted(insight_metrics): - candidate_means: dict[str, float] = {} - for candidate_label in sorted(values_by_candidate): - metric_values = values_by_candidate[candidate_label][metric_name] - candidate_mean = sum(metric_values) / len(metric_values) - profile[(candidate_label, metric_name)] = candidate_mean - candidate_means[candidate_label] = candidate_mean - repeat_spread = max(repeat_spread, max(metric_values) - min(metric_values)) - baseline_score = candidate_means[baseline.label] - winner_score = candidate_means[winner.label] - metric_improvements[metric_name] = ( - winner_score - baseline_score, - baseline_score, - winner_score, - ) - - if repeat_spread > _MAX_REPEAT_SPREAD: - return None - metric_name, (discrimination, baseline_score, winner_score) = max( - metric_improvements.items(), - key=lambda item: (item[1][0], item[0]), - ) - if baseline_score >= 1.0 or discrimination <= _MIN_DISCRIMINATION: - return None - hashes = provenance.task_hashes.get(task.id) - if hashes is None or not task.uri: - return None - try: - task_path = str(local_path_from_uri(task.uri, context=f"Insight task {task.id!r}").resolve()) - except ValueError: - return None - - return _TaskEvidence( - suggestion=InsightPromotionSuggestion( - task_id=task.id, - task_path=task_path, - suite_identity=provenance.identity, - task_content_hash=hashes["content_hash"], - verifier_hash=hashes["verifier_hash"], - metric_name=metric_name, - discrimination=discrimination, - baseline_score=baseline_score, - winner_score=winner_score, - completed_attempts=total_attempts, - total_attempts=total_attempts, - repeat_spread=repeat_spread, - candidate_count=len(suite_candidates), - ), - profile=profile, - ) - - -def _profile_distance(left: _TaskEvidence, right: _TaskEvidence) -> float: - common_keys = set(left.profile) & set(right.profile) - if not common_keys: - return 1.0 - return sum(min(abs(left.profile[key] - right.profile[key]), 1.0) for key in common_keys) / len(common_keys) - - -def select_insight_promotion_suggestions( - dataset: Dataset, - candidates: Sequence[Candidate], - *, - winner: Candidate | None = None, - limit: int = 3, -) -> list[InsightPromotionSuggestion]: - """Select repeated, complete baseline-to-winner improvements for manual review.""" - if limit <= 0 or winner is None: - return [] - provenance = insight_suite_provenance(dataset) - evaluated_candidates = [ - candidate - for candidate in candidates - if "insight" in candidate.rewards and candidate_suite_identity(candidate) == provenance.identity - ] - if len(evaluated_candidates) < 2: - return [] - baseline = next((candidate for candidate in evaluated_candidates if candidate.round == 0), None) - if baseline is None or winner not in evaluated_candidates: - return [] - - remaining = [ - evidence - for task in dataset.list_tasks() - if ( - evidence := _task_evidence( - task, - evaluated_candidates, - baseline=baseline, - winner=winner, - provenance=provenance, - ) - ) - is not None - ] - remaining.sort( - key=lambda evidence: ( - -evidence.suggestion.discrimination, - evidence.suggestion.repeat_spread, - evidence.suggestion.task_id, - ) - ) - if not remaining: - return [] - - selected = [remaining.pop(0)] - while remaining and len(selected) < limit: - ranked: list[tuple[float, _TaskEvidence]] = [ - ( - min(_profile_distance(evidence, chosen) for chosen in selected), - evidence, - ) - for evidence in remaining - ] - diversity_score, next_evidence = max( - ranked, - key=lambda item: ( - item[0], - item[1].suggestion.discrimination, - -item[1].suggestion.repeat_spread, - item[1].suggestion.task_id, - ), - ) - if diversity_score <= _MIN_DISCRIMINATION: - break - selected.append( - replace( - next_evidence, - suggestion=replace( - next_evidence.suggestion, - diversity_score=diversity_score, - ), - ) - ) - remaining.remove(next_evidence) - - return [evidence.suggestion for evidence in selected] - - -def _markdown_cell(value: str) -> str: - return value.replace("|", r"\|").replace("\n", " ") - - -def render_insight_promotion_section( - suggestions: Sequence[InsightPromotionSuggestion], -) -> str: - """Render an advisory-only final-report section.""" - lines = [ - "## Insight Suite Promotion Suggestions", - "", - ( - "Advisory adaptive/development evidence only, not independent validation evidence. " - "These tasks were not copied into the validation dataset; review them manually before " - "changing the canonical validation set." - ), - "", - ] - if not suggestions: - lines.append( - "No task had complete repeated evidence reproducing a baseline failure and showing a winner improvement." - ) - return "\n".join(lines) - - lines.extend( - [ - "| Task | Local task path | Evidence |", - "| --- | --- | --- |", - ] - ) - for suggestion in suggestions: - diversity = ( - "highest discriminative signal" - if suggestion.diversity_score is None - else f"score-profile distance {suggestion.diversity_score:.2f}" - ) - evidence = ( - f"{suggestion.metric_name} baseline {suggestion.baseline_score:.2f} → " - f"winner {suggestion.winner_score:.2f} ({suggestion.discrimination:+.2f}) across " - f"{suggestion.candidate_count} candidates; " - f"{suggestion.completed_attempts}/{suggestion.total_attempts} attempts completed; " - f"repeat spread {suggestion.repeat_spread:.2f}; " - f"suite {suggestion.suite_identity}; task {suggestion.task_content_hash}; " - f"verifier {suggestion.verifier_hash}; {diversity}" - ) - lines.append( - f"| `{_markdown_cell(suggestion.task_id)}` | " - f"`{_markdown_cell(suggestion.task_path)}` | {_markdown_cell(evidence)} |" - ) - return "\n".join(lines) - - -def _write_marked_section( - report_path: Path, - *, - rendered: str, - start_marker: str, - end_marker: str, -) -> None: - report = report_path.read_text() if report_path.exists() else "# Optimization Report\n" - section = f"{start_marker}\n{rendered}\n{end_marker}" - if start_marker in report and end_marker in report: - before, _, marked = report.partition(start_marker) - _, _, after = marked.partition(end_marker) - report = f"{before.rstrip()}\n\n{section}{after}" - else: - report = f"{report.rstrip()}\n\n{section}\n" - report_path.parent.mkdir(parents=True, exist_ok=True) - report_path.write_text(f"{report.rstrip()}\n") - - -def write_insight_promotion_section( - report_path: Path, - suggestions: Sequence[InsightPromotionSuggestion], -) -> None: - """Append or replace the advisory promotion section in the final report.""" - _write_marked_section( - report_path, - rendered=render_insight_promotion_section(suggestions), - start_marker=_REPORT_SECTION_START, - end_marker=_REPORT_SECTION_END, - ) - - -def render_insight_comparison_section( - baseline: Candidate, - winner: Candidate, - provenance: InsightSuiteProvenance, -) -> str: - """Render the deterministic baseline-versus-winner Insight comparison.""" - for candidate in (baseline, winner): - if candidate_suite_identity(candidate) != provenance.identity: - raise ValueError( - f"Candidate {candidate.label!r} Insight evidence does not match finalized suite {provenance.identity}" - ) - baseline_reward = baseline.reward("insight").metrics or {} - winner_reward = winner.reward("insight").metrics or {} - metric_names = sorted(set(baseline_reward) | set(winner_reward)) - lines = [ - "## Deterministic Insight Suite Comparison", - "", - ( - "Adaptive/development evidence only; canonical validation remains the direct " - "Pareto and winner-selection criterion." - ), - "", - (f"Suite: `{provenance.suite_path}` (suite `{provenance.identity}`; scorer `{provenance.scorer_identity}`)"), - "", - "| Metric | Baseline | Winner | Delta |", - "| --- | ---: | ---: | ---: |", - ] - for metric_name in metric_names: - baseline_value = baseline_reward.get(metric_name) - winner_value = winner_reward.get(metric_name) - if baseline_value is None or winner_value is None: - baseline_text = "—" if baseline_value is None else f"{baseline_value:.3f}" - winner_text = "—" if winner_value is None else f"{winner_value:.3f}" - delta_text = "—" - else: - baseline_text = f"{baseline_value:.3f}" - winner_text = f"{winner_value:.3f}" - delta_text = f"{winner_value - baseline_value:+.3f}" - lines.append(f"| `{_markdown_cell(metric_name)}` | {baseline_text} | {winner_text} | {delta_text} |") - return "\n".join(lines) - - -def write_insight_comparison_section( - report_path: Path, - baseline: Candidate, - winner: Candidate, - provenance: InsightSuiteProvenance, -) -> None: - """Append or replace the deterministic baseline-versus-winner section.""" - _write_marked_section( - report_path, - rendered=render_insight_comparison_section(baseline, winner, provenance), - start_marker=_COMPARISON_SECTION_START, - end_marker=_COMPARISON_SECTION_END, - ) diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py index 2e111dc889..3160188ce1 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/loop.py @@ -19,7 +19,12 @@ from typing import Any, Literal, cast, get_args from nemo_eval_author_plugin.eval_author.agent import EvalAuthor -from nemo_experimentalist_plugin.config import EvolutionaryOptimizerConfig +from nemo_experimentalist_plugin.config import ( + EvolutionaryOptimizerConfig, + MetricTarget, + has_metric_dimensions, + pareto_objectives, +) from nemo_experimentalist_plugin.entities import ( Candidate, Dataset, @@ -29,7 +34,10 @@ ) from nemo_experimentalist_plugin.experimentalist.components.analyzer import AgentAnalyzer from nemo_experimentalist_plugin.experimentalist.components.coder import Coder, CoderConfig -from nemo_experimentalist_plugin.experimentalist.components.dataset_staging import stage_eval_author_inputs +from nemo_experimentalist_plugin.experimentalist.components.dataset_staging import ( + distribute_insight_suite_tasks, + stage_eval_author_inputs, +) from nemo_experimentalist_plugin.experimentalist.components.evaluator import Evaluator from nemo_experimentalist_plugin.experimentalist.components.evaluator.factory import DatasetFactory, EvaluatorFactory from nemo_experimentalist_plugin.experimentalist.components.goal_tree import ( @@ -43,16 +51,6 @@ ensure_heldout_hidden, restore_heldout_splits, ) -from nemo_experimentalist_plugin.experimentalist.components.insight_promotion import ( - candidate_metric_keys, - candidate_suite_identity, - insight_suite_provenance, - select_insight_promotion_suggestions, - stamp_insight_evaluation_result, - validate_insight_evaluation_result, - write_insight_comparison_section, - write_insight_promotion_section, -) from nemo_experimentalist_plugin.experimentalist.components.model_config import ( get_fast_model, get_smart_model, @@ -60,10 +58,10 @@ from nemo_experimentalist_plugin.experimentalist.components.models import ( EvolutionTree, OptimizationType, - pareto_front, pareto_sort, ) from nemo_experimentalist_plugin.experimentalist.components.proposer import Improvement, Proposer +from nemo_experimentalist_plugin.experimentalist.components.selector import SurvivorSelector from nemo_experimentalist_plugin.experimentalist.components.terminator import Terminator from nemo_experimentalist_plugin.experimentalist.components.tools import ( GuardedShellTools, @@ -76,7 +74,6 @@ from nemo_experimentalist_plugin.experimentalist.experimentalist_backend import ( ExperimentalistBackend, ) -from nemo_experimentalist_plugin.experimentalist.reporting import reward_scalar from nemo_experimentalist_plugin.experimentalist.result import ExperimentalistResult from nemo_platform import AsyncNeMoPlatform from nooa import Agent, CodeActStrategy, strategy @@ -135,6 +132,28 @@ def _coerce_optimization_type(optimization_type: str | None) -> OptimizationType return None +def _with_insight_objective( + config: EvolutionaryOptimizerConfig, metric_keys: tuple[str, ...] +) -> EvolutionaryOptimizerConfig: + """Make insight metrics objectives and preserve all configured targets as guardrails.""" + if not metric_keys: + return config + insight_metric_names = set(metric_keys) + objective = [MetricTarget(name=metric_key, direction="maximize") for metric_key in metric_keys] + regression_by_name = { + target.name: target + for target in [*config.objective_function, *config.regression_metrics] + if target.name not in insight_metric_names + } + return EvolutionaryOptimizerConfig.model_validate( + config.model_dump(mode="python") + | { + "objective_function": [target.model_dump(mode="python") for target in objective], + "regression_metrics": [target.model_dump(mode="python") for target in regression_by_name.values()], + } + ) + + def _trajectory_detail_from_reward(value: Any) -> dict[str, Any]: if isinstance(value, dict): reward_val = value.get("reward", value.get("score", 0.0)) @@ -194,23 +213,6 @@ class AnalysisSkill(Skill): | agent-1 | 0.48 | 0.58 | 0.41 | ... | agent-0 | -0.10 | | agent-0 | 0.45 | 0.55 | 0.40 | ... | --- | baseline | - Insight Suite Reward: - | Agent | | | ... | vs. Baseline | - | ----- | -------------- | -------------- | --- | ------------ | - | agent-3 | 0.80 | 0.67 | ... | +0.40 | - | agent-1 | 0.60 | 0.50 | ... | +0.20 | - | agent-0 | 0.40 | 0.33 | ... | baseline | - - [Columns are the actual reward dimension keys from metadata. Order by any dimension that - helps comparison — no dimension is privileged. Read Insight Suite Reward from - `candidate.rewards["insight"].metrics`. Omit that table when the `insight` channel is - absent or empty for every agent. Keep Insight Suite Reward separate from train and validation rewards: - it reports performance on scenarios authored for the motivating Insight and is not a - ranking or Pareto-selection input. Insight Suite metrics may steer round analysis, - goal-tree updates, and the proposer only as adaptive/development feedback. Label any - resulting claim accordingly; never present this adaptive evidence as independent - validation evidence.] - ## Trajectory Rewards Trajectory rewards measure intermediate step quality (goal-tree subgoal rollup). @@ -342,6 +344,7 @@ def __init__( self._workspace_path = self.working_dir self._framework_skills_dirs: list[Path] = framework_skills_dirs or [] self.terminator = Terminator() + self.selector = SurvivorSelector(workspace=self.working_dir) self.shell = GuardedShellTools(cwd=self.working_dir) self.workspace = WorkspaceTool(workspace=self.working_dir) self.context["file_match"] = doc(Match) @@ -390,6 +393,7 @@ async def _run(self, deps: ExperimentalistDeps) -> ExperimentalistResult: backend = deps.backend workspace = deps.workspace config = deps.config if deps.config is not None else self.config + self.config = config reporter = getattr(deps, "reporter", None) # ---- Preflight: fail fast when persistence is enabled but git is missing. @@ -413,7 +417,6 @@ async def _run(self, deps: ExperimentalistDeps) -> ExperimentalistResult: train_dataset_ref = deps.train_dataset validation_dataset_ref = deps.validation_dataset task_template_ref = deps.task_template - insight_eval_dataset: Dataset | None = None if deps.insight is not None: if task_template_ref is None: raise ValueError("Task template is required for insight trace analysis") @@ -490,9 +493,21 @@ async def _run(self, deps: ExperimentalistDeps) -> ExperimentalistResult: validation_dataset=validation_eval_dataset, client=backend.client, ) + config = _with_insight_objective(config, eval_author_result.metric_keys) + self.config = config + logger.info("[METRICS] Insight objective: %s", config.optimization_policy()) train_eval_dataset = eval_author_result.train_dataset validation_eval_dataset = eval_author_result.validation_dataset - insight_eval_dataset = eval_author_result.insight_suite + if eval_author_result.insight_suite is not None: + restore_heldout_splits(self.working_dir) + try: + distribute_insight_suite_tasks( + eval_author_result.insight_suite, + train_eval_dataset, + validation_eval_dataset, + ) + finally: + ensure_heldout_hidden(self.working_dir) else: # Mode 2: local agent directory as baseline, no insight required. insight = None @@ -512,7 +527,7 @@ async def _run(self, deps: ExperimentalistDeps) -> ExperimentalistResult: None, ) if baseline_node is not None and baseline_node.val_reward: - reporter.seed_baseline(reward_scalar(baseline_node.val_reward)) + reporter.seed_baseline(baseline_node.val_reward) run_entity = self._load_run_entity() or await self._create_experiment_run( workspace=workspace, backend=backend, @@ -583,7 +598,8 @@ async def _run(self, deps: ExperimentalistDeps) -> ExperimentalistResult: reporter.candidate_evaluated( label=candidates[0].label, split="validation", - reward=reward_scalar(validation_result.aggregate_metrics), + metrics=validation_result.aggregate_metrics, + objective_metrics=config.objective_function, artifacts=self._results_dir(validation_result.id), ) except Exception: @@ -593,21 +609,6 @@ async def _run(self, deps: ExperimentalistDeps) -> ExperimentalistResult: run_id = run_entity.id or "" - if insight_eval_dataset is not None: - try: - await self._evaluate_and_persist_insight_candidates( - dataset=insight_eval_dataset, - evaluator=evaluator, - candidates=candidates, - workspace=workspace, - backend=backend, - run_id=run_entity.id or "", - ) - except Exception: - run_entity.status = "failed" - await backend.update_run(workspace=workspace, run=run_entity) - raise - # ---- Initial goal tree (idempotent) ------------------------------ await self._generate_initial_goal_tree( dataset=train_eval_dataset, @@ -637,7 +638,10 @@ async def _run(self, deps: ExperimentalistDeps) -> ExperimentalistResult: break survivors = ( - await self._select_survivors([c.slim() for c in candidates], k=config.max_survivors) + await self._select_survivors( + [c.slim() for c in candidates], + k=config.max_survivors, + ) if len(candidates) > 1 else list(candidates) ) @@ -683,7 +687,8 @@ async def _run(self, deps: ExperimentalistDeps) -> ExperimentalistResult: reporter.candidate_evaluated( label=survivor.label, split="train", - reward=reward_scalar(train_candidate_results[survivor.label].aggregate_metrics), + metrics=train_candidate_results[survivor.label].aggregate_metrics, + objective_metrics=config.objective_function, artifacts=self._results_dir(train_candidate_results[survivor.label].id), ) analysis = await self._analyze_round( @@ -747,15 +752,6 @@ async def _run(self, deps: ExperimentalistDeps) -> ExperimentalistResult: backend=backend, run_id=run_id, ) - if insight_eval_dataset is not None: - await self._evaluate_and_persist_insight_candidates( - dataset=insight_eval_dataset, - evaluator=evaluator, - candidates=new_candidates, - workspace=workspace, - backend=backend, - run_id=run_id, - ) for c in new_candidates: evolution_tree.add(c) @@ -810,7 +806,8 @@ async def _run(self, deps: ExperimentalistDeps) -> ExperimentalistResult: reporter.candidate_evaluated( label=candidate.label, split="validation", - reward=reward_scalar(validation_candidate_results[candidate.label].aggregate_metrics), + metrics=validation_candidate_results[candidate.label].aggregate_metrics, + objective_metrics=config.objective_function, artifacts=self._results_dir(validation_candidate_results[candidate.label].id), ) if not config.disable_trajectory_scoring: @@ -861,7 +858,6 @@ async def _run(self, deps: ExperimentalistDeps) -> ExperimentalistResult: run_entity=run_entity, evolution_tree=evolution_tree, agent_name=agent_name, - insight_dataset=insight_eval_dataset, ) baseline_entity = next( @@ -892,36 +888,15 @@ async def _run(self, deps: ExperimentalistDeps) -> ExperimentalistResult: _warn_persistence_failure("publish", winner_entity.label, exc) return result - @strategy(CodeActStrategy(config=CodeActConfig(max_iterations=100, cell_timeout=3600.0))) - async def select_diverse_survivors(self, ranked: list[Candidate], k: int) -> list[Candidate]: # pyright: ignore[reportReturnType] - """Choose up to k survivors from Pareto-ranked candidates. - - ``ranked`` is already Pareto-sorted using outcome and trajectory scores: - front 0 (non-dominated) first, then front 1, etc. Inside a front, - candidates are incomparable, so prefer agents with distinct architecture - changes, complementary task coverage, and different trajectory strengths. - - To avoid getting stuck on the same agents round after round: - 1. Always include at least 1 candidate that was newly created this round. Look up - `self.workspace.get_metadata(c.name)["round"]` for each candidate; new candidates - are the ones whose round equals the max round across `ranked`. - 2. Prefer candidates with different optimization_type values or that address different root causes. - 3. If all new candidates sit on a worse Pareto front, still include the best new candidate. - - ## MANDATORY: Prefer agents with complementary strengths - - Read each candidate's per-dimension validation rewards from metadata and - prefer a set whose strong dimensions cover each other (one agent leads on - dimensions where another trails): - - ```python - rewards = {c.id: self.workspace.get_metadata(c.name).reward("validation").metrics or {} for c in ranked} - ``` - - Between 1 to 3 survivors per round. - Return the selected subset, preserving the input's Pareto-front order. - """ - ... + async def select_diverse_survivors( + self, + ranked: list[Candidate], + k: int, + objective_metrics: list[MetricTarget], + regression_metrics: list[MetricTarget], + ) -> list[Candidate]: # pyright: ignore[reportReturnType] + """Backward-compatible delegate for :class:`SurvivorSelector`.""" + return await self.selector.select(ranked, k, objective_metrics, regression_metrics) @strategy(CodeActStrategy(config=CodeActConfig(max_iterations=100, cell_timeout=3600.0))) async def merge_analysis( @@ -929,34 +904,36 @@ async def merge_analysis( agent_ids: list[Candidate], round: int, per_agent_analyses: list[str], # noqa: A002 + objective_metrics: list[MetricTarget], + regression_metrics: list[MetricTarget], ) -> str: # pyright: ignore[reportReturnType] """Merge per-agent analyses, compare agents, and write the round analysis file. ## Step 1: Compare agents at reward level + Active objectives: ``objective_metrics``. Active regression guardrails: + ``regression_metrics``. Analyze objective improvements and shortfalls first; + discuss regression only as a constraint on objective improvement. + Treat evaluator metrics (including aggregate metrics) as authoritative; + do not derive a scalar score or prescribe a selection algorithm. + Read each agent's per-dimension train rewards from metadata: ```python rewards = {c.id: self.workspace.get_metadata(c.name).reward("train").metrics or {} for c in agent_ids} - insight_rewards = { - c.id: self.workspace.get_metadata(c.name).reward("insight").metrics or {} for c in agent_ids - } - all_candidates = [ - self.workspace.get_metadata(agent_id).slim() for agent_id in self.workspace.list_agents() - ] - baseline = next((candidate for candidate in all_candidates if candidate.round == 0), None) ``` - Compare siblings: which optimization strategy worked better this round? - Compare to ancestors: did the change actually fix the targeted root cause? - - When any Insight Suite rewards are present, compare those dimensions to the - round-zero baseline separately from train and validation rewards. ## Step 2: Analyze divergent and complementary patterns Using the per-dimension rewards above, identify where agents diverge (one leads, another trails on a dimension) and where their strengths are - complementary. Ground observations in the per-agent analyses passed in. + complementary. Prioritize divergences on objective metrics. A regression + metric can identify a guardrail risk, but must not become the primary + failure pattern or root cause. Ground observations in the per-agent + analyses passed in. ## Step 3: Build the round analysis markdown @@ -965,21 +942,16 @@ async def merge_analysis( candidate = self.workspace.get_metadata(agent_ids[0].name).slim() train_reward = candidate.reward("train").metrics or {} dim_keys = sorted(train_reward.keys()) - insight_dim_keys = sorted({key for reward in insight_rewards.values() for key in reward}) ``` Follow the `ext.analysis_skill` format exactly for every section (Rewards tables, - including the conditional Insight Suite Reward table; Trajectory Rewards; Divergent - Trial Analysis; Complementary Failures; Failure Patterns; Root Causes; + Trajectory Rewards; Divergent Trial Analysis; Complementary Failures; Failure Patterns; Root Causes; Mechanical/Infrastructure Errors). - If at least one agent has a non-empty `insight_rewards` entry, the round analysis must name - every available Insight Suite dimension and show its values in the separate Insight - Suite Reward table. Never blend those metrics into train/validation rewards or imply - that they affected ranking. These metrics may steer this analysis, the goal tree, and - the proposer only as adaptive/development feedback; label claims accordingly and never - present them as independent validation evidence. Fill in every included section with - real data. No placeholders. + Fill in every included section with real data. No placeholders. + Every Failure Pattern and Root Cause must explain a failing or + underperforming objective metric. Do not write a root-cause narrative + solely about a regression metric; record it as a regression risk instead. Return the complete markdown content as a string. """ ... @@ -993,7 +965,6 @@ async def write_final_report(self, best_agent_id: str) -> None: # pyright: igno ```python agent_ids = self.workspace.list_agents() candidate = self.workspace.get_metadata(agent_id).slim() - insight_reward = candidate.reward("insight").metrics or {} analysis = self.workspace.read_analysis_file(n) ``` @@ -1004,17 +975,10 @@ async def write_final_report(self, best_agent_id: str) -> None: # pyright: igno 4. Write eval-and-optimize/OPTIMIZATION.md with format: - Summary (baseline vs best rewards, rounds completed, total agents) - Reward Breakdown table (one row per agent, per-dimension columns) - - Insight Suite Metrics table when available - Lineage Tree (ASCII tree with rewards and optimization type) - Round-by-Round Analysis - Optimization Insights - When both the round-zero baseline and best agent have non-empty `insight_reward`, - the Summary must state whether the Insight-specific scenarios improved and the - Insight Suite Metrics table must show every available dimension with baseline, - winner, and signed delta columns. Keep this table separate from generic train and - validation rewards. Omit it only when Insight Suite rewards are unavailable. - Fill in every section with real data. Every agent must appear in the lineage tree. Mark the best agent with * BEST. """ @@ -1323,7 +1287,6 @@ async def _evaluate_agent( dataset: Dataset, evaluator: Evaluator, task_ids: list[str] | None = None, - minimum_attempts: int | None = None, ) -> tuple[Candidate, EvaluationResult]: """Run evaluator for one candidate and return the candidate/result pair.""" eval_dataset = dataset.subset(task_ids) if task_ids is not None else dataset @@ -1331,11 +1294,6 @@ async def _evaluate_agent( # collide on the same results directory when the user sets a fixed job_name. options_dict = evaluator.options.model_dump() options_dict["job_name"] = f"{candidate.label}-{eval_dataset.id}" - if minimum_attempts is not None: - configured_attempts = options_dict.get("n_attempts") - if not isinstance(configured_attempts, int): - raise ValueError("Insight evaluator options must define integer n_attempts") - options_dict["n_attempts"] = max(configured_attempts, minimum_attempts) per_candidate_options = type(evaluator.options).model_validate(options_dict) result = await evaluator.run( agent=self.working_dir / "eval-and-optimize" / "agents" / candidate.label, @@ -1422,106 +1380,6 @@ async def _evaluate_validation_candidates( if candidate_result is not None } - async def _evaluate_insight_candidates( - self, - *, - dataset: Dataset, - evaluator: Evaluator, - candidates: list[Candidate], - ) -> dict[str, EvaluationResult]: - """Evaluate candidates that do not yet have metrics for this Insight suite.""" - if not list(dataset.list_tasks()): - return {} - provenance = insight_suite_provenance(dataset) - pending = [ - candidate - for candidate in candidates - # One channel-presence check replaces the old pair of `insight_reward is None` - # / `insight_reward_details is None`: a RewardRecord carries metrics and trials - # together, and an empty `trials` is valid cached state, not a missing measurement. - if "insight" not in candidate.rewards - or candidate_suite_identity(candidate) != provenance.identity - or not candidate_metric_keys(candidate) - ] - evaluated = await asyncio.gather( - *[ - self._evaluate_agent( - candidate, - dataset, - evaluator, - minimum_attempts=2, - ) - for candidate in pending - ] - ) - return {candidate.label: result for candidate, result in evaluated} - - async def _evaluate_and_persist_insight_candidates( - self, - *, - dataset: Dataset, - evaluator: Evaluator, - candidates: list[Candidate], - workspace: str, - backend: ExperimentalistBackend, - run_id: str, - ) -> None: - """Evaluate and persist Insight-suite metrics for the supplied candidates.""" - provenance = insight_suite_provenance(dataset) - results = await self._evaluate_insight_candidates( - dataset=dataset, - evaluator=evaluator, - candidates=candidates, - ) - dataset_metric_keys = dataset.metadata.get("insight_metric_keys") - if dataset_metric_keys is not None and ( - not isinstance(dataset_metric_keys, list) or not all(isinstance(key, str) for key in dataset_metric_keys) - ): - raise ValueError("Insight suite runtime metric keys have invalid metadata") - cached_metric_key_sets = { - tuple(sorted(candidate_metric_keys(candidate))) - for candidate in candidates - if candidate_suite_identity(candidate) == provenance.identity and candidate_metric_keys(candidate) - } - if isinstance(dataset_metric_keys, list): - cached_metric_key_sets.add(tuple(sorted(dataset_metric_keys))) - if len(cached_metric_key_sets) > 1: - raise ValueError( - f"Cached Insight evaluations disagree on required metric keys: {sorted(cached_metric_key_sets)}" - ) - expected_metric_keys = next(iter(cached_metric_key_sets), None) - for candidate in candidates: - result = results.get(candidate.label) - if result is None: - continue - metric_keys = validate_insight_evaluation_result( - result, - expected_metric_keys=expected_metric_keys, - ) - if expected_metric_keys is None: - expected_metric_keys = metric_keys - result = stamp_insight_evaluation_result(result, provenance) - await backend.persist_evaluation( - workspace=workspace, - result=result, - candidate=candidate, - split="insight", - ) - candidate.record_reward( - "insight", - metrics=result.aggregate_metrics, - trials=result.trials, - metadata={"suite_identity": provenance.identity, "metric_keys": list(metric_keys)}, - ) - await self._update_candidate( - candidate, - workspace=workspace, - backend=backend, - run_id=run_id, - ) - if expected_metric_keys is not None: - dataset.metadata["insight_metric_keys"] = list(expected_metric_keys) - async def _generate_initial_goal_tree( self, *, @@ -1553,9 +1411,29 @@ async def _select_survivors( candidates: list[Candidate], k: int, ) -> list[Candidate]: - """Return the top-k Pareto-optimal and architecturally diverse candidates.""" - ranked = pareto_sort(candidates, lambda c: c.reward("validation").metrics or {}) - return await self.select_diverse_survivors(ranked, k) + """Return diverse Pareto survivors, with regression metrics as selector context.""" + eligible = [ + candidate + for candidate in candidates + if has_metric_dimensions(candidate.reward("validation").metrics or {}, self.config.objective_function) + ] + if not eligible: + raise ValueError( + "No candidate reports every objective metric: " + f"{[target.name for target in self.config.objective_function]}" + ) + ranked = pareto_sort( + eligible, + lambda candidate: pareto_objectives( + candidate.reward("validation").metrics or {}, self.config.objective_function + ), + ) + return await self.select_diverse_survivors( + ranked, + k, + self.config.objective_function, + self.config.regression_metrics, + ) async def _evaluate_train_candidates( self, @@ -1651,11 +1529,19 @@ async def _analyze_round( client=client, nmp_workspace=nmp_workspace, agent_spec=agent_spec_path, + objective_metrics=[target.model_dump() for target in config.objective_function], + regression_metrics=[target.model_dump() for target in config.regression_metrics], ) for s in survivors ] ) - analysis = await self.merge_analysis(survivors, round_num, [str(a) for a in per_agent]) + analysis = await self.merge_analysis( + survivors, + round_num, + [str(a) for a in per_agent], + config.objective_function, + config.regression_metrics, + ) analysis_path.parent.mkdir(parents=True, exist_ok=True) analysis_path.write_text(analysis) return analysis @@ -1712,6 +1598,8 @@ async def _propose_improvements( round_num=round_num, phase=phase, max_candidates=config.max_candidates, + objective_metrics=[target.model_dump(mode="json") for target in config.objective_function], + regression_metrics=[target.model_dump(mode="json") for target in config.regression_metrics], ) async def _implement_candidates( @@ -1896,13 +1784,22 @@ async def _finalize( run_entity: ExperimentRun, evolution_tree: EvolutionTree, agent_name: str, - insight_dataset: Dataset | None, ) -> Candidate | None: """Select the winner, copy to workspace root, write final report.""" # Only survivors that actually have a validation reward are eligible winners. scored = [n for n in evolution_tree.nodes.values() if n.is_survivor and n.val_reward] - front = pareto_front(scored, lambda n: n.val_reward) if scored else [] - best_id = front[0].label if front else None + eligible = [node for node in scored if has_metric_dimensions(node.val_reward, self.config.objective_function)] + ranked_nodes = pareto_sort( + eligible, + lambda node: pareto_objectives(node.val_reward, self.config.objective_function), + ) + finalists = await self.select_diverse_survivors( + [node.candidate for node in ranked_nodes], + 1, + self.config.objective_function, + self.config.regression_metrics, + ) + best_id = finalists[0].label if finalists else None restore_heldout_splits(self.working_dir) @@ -1937,28 +1834,6 @@ async def _finalize( ) report_path.write_text(f"# Optimization Report\n\n## Compact Run Summary\n\n{summary}\n") - if insight_dataset is not None: - try: - provenance = insight_suite_provenance(insight_dataset) - if baseline is not None: - write_insight_comparison_section( - report_path, - baseline, - winner, - provenance, - ) - suggestions = select_insight_promotion_suggestions( - insight_dataset, - [node.candidate for node in evolution_tree.nodes.values()], - winner=winner, - ) - write_insight_promotion_section( - report_path, - suggestions, - ) - except ValueError as exc: - logger.warning(f"[FINAL] Skipping Insight Suite report sections: {exc}") - run_entity.status = "completed" run_entity.winner_agent = best_id await backend.update_run(workspace=workspace, run=run_entity) @@ -1976,10 +1851,6 @@ def _render_summary( details: list[str] = [] if winner: if winner.reward("validation").metrics: - details.append(f"validation_reward={winner.reward('validation').metrics}") - if baseline is not None and baseline.reward("insight").metrics and winner.reward("insight").metrics: - details.append( - f"insight_suite=(baseline={baseline.reward('insight').metrics}, winner={winner.reward('insight').metrics})" - ) + details.append(f"validation_metrics={winner.reward('validation').metrics}") suffix = f", {', '.join(details)}" if details else "" return f"Optimization complete: {rounds_completed} round(s) completed, winner={winner_str}{suffix}" diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/proposer.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/proposer.py index 0965f224ea..ea14766328 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/proposer.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/proposer.py @@ -101,6 +101,8 @@ async def run( round_num: int, phase: Literal["exploration", "exploitation"], max_candidates: int, + objective_metrics: list[dict[str, str]], + regression_metrics: list[dict[str, str]], ) -> list[Improvement]: """Return up to max_candidates targeted improvement proposals. @@ -111,6 +113,8 @@ async def run( round_num: current optimization round number; used to filter survivors. phase: "exploration" for novel directions, "exploitation" to refine the best. max_candidates: maximum number of Improvement objects to return. + objective_metrics: Evaluator metric dimensions this round must improve. + regression_metrics: Evaluator metric dimensions this round must preserve. Returns: list[Improvement]: up to max_candidates targeted improvement proposals. @@ -143,7 +147,7 @@ async def run( survivor_context.append( { "id": s.label, - "reward": s.reward("validation").metrics or {}, + "metrics": s.reward("validation").metrics or {}, "trajectory_reward": s.reward("validation-trajectory").metrics or {}, "metadata": meta, "architecture": arch_text, @@ -159,6 +163,8 @@ async def run( cards_index=doc(self.optimize), phase=phase, max_candidates=max_candidates, + objective_metrics=objective_metrics, + regression_metrics=regression_metrics, ) # allowed_types is every type, not just the untried ones. available_types # still reaches the prompt, so novelty stays a *preference*: a type that @@ -236,6 +242,8 @@ async def _run_with_context( cards_index: str, phase: Literal["exploration", "exploitation"], max_candidates: int, + objective_metrics: list[dict[str, str]], + regression_metrics: list[dict[str, str]], ) -> list[Improvement]: """Pick up to `max_candidates` targeted improvements grounded in root causes. @@ -244,13 +252,15 @@ async def _run_with_context( - evolution_history (str): markdown table of prior rounds, for context - tried_types (list[str]): optimization_types already attempted - available_types (list[str]): types not yet tried — PREFER THESE - - survivors (list[dict]): each {id, reward, trajectory_reward, metadata, + - survivors (list[dict]): each {id, metrics, trajectory_reward, metadata, architecture}; these are your branching candidates with their architecture.md already loaded - cards_index (str): pre-rendered `doc(self.optimize)` showing the card index. Load a specific card on demand via `print(doc(self.optimize.))`. - phase (Literal): "exploration" = novel directions; "exploitation" = improve current best - max_candidates (int): max number of Improvements to return (typically 3) + - objective_metrics (list[dict]): evaluator dimensions to improve + - regression_metrics (list[dict]): evaluator dimensions to preserve Returns: - list[Improvement]: up to `max_candidates` Improvements. @@ -263,7 +273,8 @@ async def _run_with_context( ## Per-improvement requirements For each Improvement you propose: - 1. Identify ONE root cause from the analysis: the specific reason the agent + 1. Identify ONE root cause from the analysis that limits an active objective: + the specific reason the agent underperforms, stated as a diagnosis ("The agent fails because X is absent / misconfigured / too vague"). Do NOT include the proposed remedy here — that belongs in `optimization`. If you cannot articulate the failure cause @@ -285,6 +296,17 @@ async def _run_with_context( validate the fix. Pick 2-3 tasks the ancestor actually fails (or barely passes) for this reason; do not pad with unrelated passing tasks. + ## Metric contract + - Optimize these objective metrics: `objective_metrics`. Every proposed + change must have a concrete, evidence-based path to improving at least one + of these dimensions. + - Preserve these regression metrics: `regression_metrics`. They are + guardrails, not proposal targets: do not choose a root cause or frame an + optimization solely around improving a regression metric. Instead, ensure + the proposed objective improvement does not sacrifice them. + - Evaluator metric values are authoritative. Do not invent an aggregate, + scalarization, weighting, or threshold. + ## Branching rules - Branch from any survivor — usually the top scorer, but a lower-scoring ancestor is the right base when it uniquely passes tasks the top scorer fails. diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/selector.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/selector.py new file mode 100644 index 0000000000..511425cc39 --- /dev/null +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/selector.py @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The reasoning component that selects a diverse survivor set.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from nemo_experimentalist_plugin.config import MetricTarget +from nemo_experimentalist_plugin.entities import Candidate +from nemo_experimentalist_plugin.experimentalist.components.model_config import get_smart_model +from nemo_experimentalist_plugin.experimentalist.components.tools import WorkspaceTool +from nooa import Agent, CodeActStrategy, strategy +from nooa.config import CodeActConfig + + +class SurvivorSelector(Agent): + """Select survivors from a Pareto-ranked population.""" + + def __init__(self, workspace: Path, **kwargs: Any) -> None: + super().__init__(llm=kwargs.pop("llm", None) or get_smart_model(), **kwargs) + self.workspace = WorkspaceTool(workspace=workspace) + + @strategy(CodeActStrategy(config=CodeActConfig(max_iterations=100, cell_timeout=3600.0))) + async def select( + self, + ranked: list[Candidate], + k: int, + objective_metrics: list[MetricTarget], + regression_metrics: list[MetricTarget], + ) -> list[Candidate]: # pyright: ignore[reportReturnType] + """Choose up to k survivors from Pareto-ranked candidates. + + ``ranked`` is already Pareto-sorted using outcome and trajectory scores: + front 0 (non-dominated) first, then front 1, etc. Inside a front, + candidates are incomparable, so prefer agents with distinct architecture + changes, complementary task coverage, and different trajectory strengths. + + To avoid getting stuck on the same agents round after round: + 1. Always include at least 1 candidate that was newly created this round. Look up + `self.workspace.get_metadata(c.name)["round"]` for each candidate; new candidates + are the ones whose round equals the max round across `ranked`. + 2. Prefer candidates with different optimization_type values or that address different root causes. + 3. If all new candidates sit on a worse Pareto front, still include the best new candidate. + + ## MANDATORY: Prefer agents with complementary strengths + + Read each candidate's per-dimension validation rewards from metadata and + prefer a set whose strong dimensions cover each other (one agent leads on + dimensions where another trails): + + ```python + rewards = {c.id: self.workspace.get_metadata(c.name).reward("validation").metrics or {} for c in ranked} + ``` + + Between 1 to 3 survivors per round. + Return the selected subset, preserving the input's Pareto-front order. + + ``objective_metrics`` are the evaluator metric dimensions to improve. + ``regression_metrics`` are evaluator metric dimensions that must not + worsen. Use their reported values as evidence; do not invent formulas, + weights, thresholds, or another mechanical selection rule. + """ + ... diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/terminator.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/terminator.py index 37c9571765..623051a6d3 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/terminator.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/terminator.py @@ -16,7 +16,12 @@ # Imported from `resolve` rather than `.loop`, which merely re-exports it: `loop` imports # this module, so going through it would be circular. -from nemo_experimentalist_plugin.config import EvolutionaryOptimizerConfig +from nemo_experimentalist_plugin.config import ( + EvolutionaryOptimizerConfig, + MetricTarget, + has_metric_dimensions, + pareto_objectives, +) from nemo_experimentalist_plugin.experimentalist.components.models import EvolutionTree, pareto_front from nemo_experimentalist_plugin.skills import skills_dir from nooa import Agent, CodeActStrategy, TextSkill, hidden, strategy @@ -110,6 +115,8 @@ async def assess_convergence( evolution_tree=evolution_tree, prior_analysis=prior_analysis, min_rounds_before_stopping=config.min_rounds_before_stopping, + objective_metrics=config.objective_function, + regression_metrics=config.regression_metrics, ) if converged: return TerminationDecision(stop=True, reason="optimization converged (Pareto front stagnated)") @@ -143,6 +150,8 @@ async def _has_converged( evolution_tree: EvolutionTree, prior_analysis: str, min_rounds_before_stopping: int, + objective_metrics: list[MetricTarget] | None = None, + regression_metrics: list[MetricTarget] | None = None, ) -> bool: """Return True if the optimization has converged and should stop early.""" # Truthy check (not ``is not None``): ``EvolutionNode.val_reward`` returns ``{}`` for @@ -156,19 +165,46 @@ async def _has_converged( old = [n for n in scored if n.round <= cutoff_round] if not old: return False - old_front_ids = {n.label for n in pareto_front(old, lambda n: n.val_reward)} - full_front_ids = {n.label for n in pareto_front(scored, lambda n: n.val_reward)} + active_objectives = objective_metrics or EvolutionaryOptimizerConfig().objective_function + active_regressions = regression_metrics or [] + scored = [node for node in scored if has_metric_dimensions(node.val_reward, active_objectives)] + if not scored: + return False + old = [node for node in old if node in scored] + if not old: + return False + old_front_ids = { + node.label for node in pareto_front(old, lambda node: pareto_objectives(node.val_reward, active_objectives)) + } + full_front_ids = { + node.label + for node in pareto_front(scored, lambda node: pareto_objectives(node.val_reward, active_objectives)) + } if full_front_ids.issubset(old_front_ids): return True - return await self.qualitative_stop_check(prior_analysis) + return await self.qualitative_stop_check( + prior_analysis, + active_objectives, + active_regressions, + ) @strategy(CodeActStrategy(config=CodeActConfig(max_iterations=5))) - async def qualitative_stop_check(self, analysis: str) -> bool: # pyright: ignore[reportReturnType] + async def qualitative_stop_check( + self, + analysis: str, + objective_metrics: list[MetricTarget], + regression_metrics: list[MetricTarget], + ) -> bool: # pyright: ignore[reportReturnType] """Decide whether the optimization has qualitatively plateaued; return True to stop. Judge the round ``analysis`` text against the terminator skill's stop heuristics (prefilled below). Return True only with concrete textual evidence of stagnation in ``analysis``; when in doubt, return False. + + ``objective_metrics`` are the evaluator metrics being improved; + ``regression_metrics`` are those that must not worsen. Judge stagnation + only in that context; do not invent a scalar score, weights, or a new + selection rule. """ print(doc(self.terminator_skill)) ... diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_analyzer.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_analyzer.py index b23f2d7781..433a7091b7 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_analyzer.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/components/trace_analyzer.py @@ -193,6 +193,9 @@ async def analyze_trajectory( agent_path: Path, runtime: DependencyRuntime | None, insight: Insight | None = None, + selection_reason: str = "", + objective_metrics: list[dict[str, str]] | None = None, + regression_metrics: list[dict[str, str]] | None = None, ) -> StepAnalysis: """Trace the agent's path and find where it went wrong. @@ -204,6 +207,10 @@ async def analyze_trajectory( rationale (Rationale): reference solution; may have empty steps insight: Production insight. If present, use its title and description as the analysis lens. + selection_reason: Why the trial was selected for in-depth analysis. + objective_metrics: Evaluator metric dimensions the optimization run + aims to improve. + regression_metrics: Evaluator metric dimensions that must not worsen. agent_path: Local path to the agent root. runtime: Dependency runtime to use for analyzing the trace. @@ -215,6 +222,14 @@ async def analyze_trajectory( If insight is present, look specifically for the behavior described by ``insight.title`` and ``insight.description``. If ``agent_path`` is present, inspect agent code there when it helps explain the trace. + Use ``selection_reason`` to focus on the evidence that motivated this + analysis. Interpret the trace and ``trial.metrics`` against + ``objective_metrics`` (dimensions to improve) and ``regression_metrics`` + (dimensions to preserve). Do not invent scalar formulas, weights, or + thresholds for evaluator-defined metrics. + Frame decision points around the objective metric shortfall. A regression + metric may identify a guardrail risk, but is not by itself the failure this + optimization round is trying to diagnose. All TraceExplorer methods are async — always `await` them. @@ -244,6 +259,8 @@ async def diagnose( trace: TraceExplorer, analysis: StepAnalysis, insight: Insight | None = None, + objective_metrics: list[dict[str, str]] | None = None, + regression_metrics: list[dict[str, str]] | None = None, ) -> Diagnostic: """Determine the primary root cause and produce a Diagnostic. @@ -252,6 +269,9 @@ async def diagnose( analysis (StepAnalysis): result from analyze_trajectory insight: Optional production insight. If present, diagnose the root cause of that insight's failure behavior. + objective_metrics: Evaluator metric dimensions the optimization run + aims to improve. + regression_metrics: Evaluator metric dimensions that must not worsen. Returns: Diagnostic: result with outcome, summary, failure_point, and root_cause. @@ -267,6 +287,10 @@ async def diagnose( Return a complete Diagnostic with outcome, summary, failure_point, and root_cause. If insight is present and the trace does not match the insight's behavior, set ``outcome="SUCCESS"`` with ``failure_point=None``. + Use the metric contract to distinguish objective shortfalls from regression + risks. The root cause must explain an objective metric's underperformance; + describe regression effects only as constraints. Do not invent scalar + formulas, weights, or thresholds. """ # noqa: D413 ... @@ -277,6 +301,9 @@ async def run( agent_path: Path, rationale: Rationale | None = None, insight: Insight | None = None, + selection_reason: str = "", + objective_metrics: list[dict[str, str]] | None = None, + regression_metrics: list[dict[str, str]] | None = None, client: AsyncNeMoPlatform | None = None, workspace: str | None = None, ) -> Diagnostic: @@ -289,6 +316,10 @@ async def run( rationale: Optional reference solution produced by Rationalizer; used as a compass for trajectory analysis. Pass ``None`` to skip comparison. insight: Optional production insight to use as the failure lens. + selection_reason: Why the analyzer selected this trial for inspection. + objective_metrics: Evaluator metric dimensions the optimization run + aims to improve. + regression_metrics: Evaluator metric dimensions that must not worsen. client: Existing NeMo Platform client for Intake trace identifiers. workspace: NMP workspace request context for Intake trace identifiers. @@ -326,7 +357,11 @@ async def run( # only ever diagnosed through one lens and cannot collide here. If a future # caller analyzes the same trace under different insights in one experiment_dir, # fold the insight identity into this key to avoid returning a stale-lens Diagnostic. - key = _trace_cache_key(trial.trace.uri) + metric_contract_key = ( + f"{_trace_cache_key(trial.trace.uri)}:selection-reason:{selection_reason}:" + f"objective-metrics:{objective_metrics or []}:regression-metrics:{regression_metrics or []}" + ) + key = cache.trace_uri_hash(metric_contract_key) cached = cache.load(self._experiment_dir, key, Diagnostic) if cached is not None: @@ -343,9 +378,18 @@ async def run( overview=overview, rationale=rationale, insight=insight, + selection_reason=selection_reason, + objective_metrics=objective_metrics, + regression_metrics=regression_metrics, agent_path=agent_path, runtime=runtime, ) - diagnostic = await self.diagnose(trace, analysis, insight) + diagnostic = await self.diagnose( + trace, + analysis, + insight, + objective_metrics, + regression_metrics, + ) cache.store(self._experiment_dir, key, diagnostic) return diagnostic diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experimentalist_backend.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experimentalist_backend.py index cd06a59167..50c4d013e5 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experimentalist_backend.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/experimentalist_backend.py @@ -507,8 +507,8 @@ async def _compose_pr_body(self, *, workspace: str, candidate: Candidate) -> str if sib.label == _BASELINE_AGENT_LABEL: continue marker = " (winner)" if sib.label == candidate.label else "" - reward = sib.reward("validation").metrics or sib.reward("train").metrics or {} - lines.append(f"- `{sib.label}`{marker}: `{self._candidate_branch(sib)}` — reward={reward}") + metrics = sib.reward("validation").metrics or sib.reward("train").metrics or {} + lines.append(f"- `{sib.label}`{marker}: `{self._candidate_branch(sib)}` — metrics={metrics}") return summary + "\n" + "\n".join(lines) + "\n" # -- Agent reads --------------------------------------------------------- diff --git a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/reporting.py b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/reporting.py index 0601a466cf..17366d3676 100644 --- a/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/reporting.py +++ b/plugins/nemo-experimentalist/src/nemo_experimentalist_plugin/experimentalist/reporting.py @@ -17,6 +17,8 @@ from pathlib import Path from typing import TextIO +from nemo_experimentalist_plugin.config import MetricTarget + _RULE = "═" * 62 _THIN = "─" * 62 @@ -33,11 +35,6 @@ class Verbosity(str, Enum): NORMAL = "normal" -def reward_scalar(aggregate_metrics: dict[str, float | int]) -> float: - """Extract the scalar reward from an evaluation's aggregate metrics.""" - return float(aggregate_metrics.get("reward", 0.0)) - - class RunReporter: """Formats run-progress narration to a sink. Never raises into the run.""" @@ -49,7 +46,8 @@ def __init__( ) -> None: self._sink: TextIO = sink if sink is not None else sys.stderr self._verbosity = verbosity - self._baseline_val: float | None = None + self._baseline_metrics: dict[str, float] | None = None + self._objective_metrics: list[MetricTarget] = [] def _emit(self, line: str) -> None: try: @@ -58,7 +56,7 @@ def _emit(self, line: str) -> None: except Exception: # noqa: BLE001 - narration must never break the run pass - def seed_baseline(self, reward: float) -> None: + def seed_baseline(self, metrics: dict[str, float | int]) -> None: """Set the validation delta reference without emitting a line. Used on resume, where agent-0 is not re-evaluated: without this, the @@ -67,8 +65,8 @@ def seed_baseline(self, reward: float) -> None: baseline is already set. """ try: - if self._baseline_val is None: - self._baseline_val = reward + if self._baseline_metrics is None: + self._baseline_metrics = {name: float(value) for name, value in metrics.items()} except Exception: # noqa: BLE001 pass @@ -109,32 +107,65 @@ def candidate_started(self, *, label: str, optimization: str, i: int | None, n: except Exception: # noqa: BLE001 pass - def candidate_evaluated(self, *, label: str, split: str, reward: float, artifacts: Path) -> None: + def candidate_evaluated( + self, + *, + label: str, + split: str, + metrics: dict[str, float | int], + objective_metrics: list[MetricTarget], + artifacts: Path, + ) -> None: + """Narrate an evaluation using all configured objective metrics. + + Objective directions determine whether validation deltas are displayed + as improvements or declines. Regression metrics are guardrails and are + intentionally not presented as progress dimensions. + """ if self._verbosity is Verbosity.QUIET: return try: - delta = "" + self._objective_metrics = objective_metrics + has_baseline = self._baseline_metrics is not None if split == "validation": - if self._baseline_val is None: - self._baseline_val = reward - else: - d = reward - self._baseline_val - delta = f" {'▲' if d >= 0 else '▼'}{d:+.3f}" - self._emit(f" {label} · {split:<10} · reward {reward:.3f}{delta} → {artifacts}") + self.seed_baseline(metrics) + rendered_metrics: list[str] = [] + for target in objective_metrics: + value = metrics.get(target.name) + if value is None: + rendered_metrics.append(f"{target.name} n/a") + continue + rendered = f"{target.name} {float(value):.3f}" + baseline_value = (self._baseline_metrics or {}).get(target.name) + if split == "validation" and has_baseline and baseline_value is not None: + delta = float(value) - baseline_value + improved = delta >= 0 if target.direction == "maximize" else delta <= 0 + rendered += f" {'▲' if improved else '▼'}{delta:+.3f}" + rendered_metrics.append(rendered) + self._emit(f" {label} · {split:<10} · {', '.join(rendered_metrics)} → {artifacts}") except Exception: # noqa: BLE001 pass - def run_finished(self, *, winner: str | None, scores: dict[str, float], report_path: Path | None) -> None: + def run_finished( + self, + *, + winner: str | None, + scores: dict[str, float], + report_path: Path | None, + ) -> None: try: self._emit(_THIN) if winner is None: self._emit(" Finished · no winner (no scored candidates)") else: head = f" Finished · winner={winner}" - val = scores.get("reward") - if val is not None: - base = f" (baseline {self._baseline_val:.3f})" if self._baseline_val is not None else "" - head += f" · validation {val:.3f}{base}" + rendered_metrics = [ + f"{target.name} {float(scores[target.name]):.3f}" + for target in self._objective_metrics + if target.name in scores + ] + if rendered_metrics: + head += f" · validation {', '.join(rendered_metrics)}" self._emit(head) if report_path is not None: self._emit(f" report: {report_path}") diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging.py b/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging.py index 7e85939a59..3d91189e2b 100644 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging.py +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging.py @@ -1,16 +1,19 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import tomllib from pathlib import Path from types import SimpleNamespace from typing import cast import pytest -from nemo_experimentalist_plugin.entities import DatasetRef +from nemo_experimentalist_plugin.entities import Dataset, DatasetRef, Task from nemo_experimentalist_plugin.experimentalist.components.dataset_staging import ( + distribute_insight_suite_tasks, stage_eval_author_inputs, stage_task_template, ) +from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import HarborDataset from nemo_platform import AsyncNeMoPlatform @@ -20,6 +23,70 @@ def _write_tree(root: Path, content: str) -> None: (tests / "test.sh").write_text(content, encoding="utf-8") +class _MemoryDataset(Dataset): + def add_tasks(self, tasks: list[Task]) -> None: + self.tasks.extend(tasks) + + +def test_distribute_insight_suite_tasks_uses_a_30_70_validation_train_split() -> None: + insight_suite = _MemoryDataset(id="insight", tasks=[Task(id=f"task-{index}") for index in range(10)]) + train_dataset = _MemoryDataset(id="train") + validation_dataset = _MemoryDataset(id="validation") + + distribute_insight_suite_tasks(insight_suite, train_dataset, validation_dataset) + + assert [task.id for task in validation_dataset.list_tasks()] == ["task-0", "task-1", "task-2"] + assert [task.id for task in train_dataset.list_tasks()] == [ + "task-3", + "task-4", + "task-5", + "task-6", + "task-7", + "task-8", + "task-9", + ] + + +def test_harbor_dataset_add_tasks_copies_task_directories(tmp_path: Path) -> None: + source_root = tmp_path / "insight-suite" + destination_root = tmp_path / "train" + (source_root / "insight-task" / "task.toml").parent.mkdir(parents=True) + (source_root / "insight-task" / "task.toml").write_text("", encoding="utf-8") + (destination_root / "insight-task" / "task.toml").parent.mkdir(parents=True) + (destination_root / "insight-task" / "stale.txt").write_text("old", encoding="utf-8") + (destination_root / "insight-task" / "task.toml").write_text("", encoding="utf-8") + insight_suite = HarborDataset.from_path(source_root) + train_dataset = HarborDataset.from_path(destination_root) + + train_dataset.add_tasks(list(insight_suite.list_tasks())) + + assert (destination_root / "insight-task" / "task.toml").is_file() + assert not (destination_root / "insight-task" / "stale.txt").exists() + assert [task.id for task in train_dataset.list_tasks()] == ["insight-task"] + assert Path(train_dataset.get_task("insight-task").uri.removeprefix("file://")) == destination_root / "insight-task" + + +def test_harbor_dataset_add_tasks_preserves_existing_task_when_replacement_is_invalid(tmp_path: Path) -> None: + source_root = tmp_path / "insight-suite" + destination_root = tmp_path / "train" + source_task = source_root / "task-1" + destination_task = destination_root / "task-1" + source_task.mkdir(parents=True) + destination_task.mkdir(parents=True) + (source_task / "task.toml").write_text("", encoding="utf-8") + (destination_task / "task.toml").write_text("", encoding="utf-8") + (destination_task / "stale.txt").write_text("keep", encoding="utf-8") + insight_suite = HarborDataset.from_path(source_root) + train_dataset = HarborDataset.from_path(destination_root) + (source_task / "task.toml").write_text("not valid = [", encoding="utf-8") + + with pytest.raises(tomllib.TOMLDecodeError): + train_dataset.add_tasks(list(insight_suite.list_tasks())) + + assert (destination_task / "stale.txt").read_text(encoding="utf-8") == "keep" + assert (destination_task / "task.toml").is_file() + + @pytest.mark.asyncio async def test_stage_eval_author_inputs_isolates_all_mutable_sources(tmp_path: Path) -> None: source = tmp_path / "source" diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging_loop.py b/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging_loop.py index d2207f3f48..fcb76a613b 100644 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging_loop.py +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_dataset_staging_loop.py @@ -21,7 +21,7 @@ def _write_tree(root: Path, content: str) -> None: @pytest.mark.asyncio -async def test_insight_run_stages_inputs_before_eval_author( +async def test_insight_run_stages_inputs_and_stops_at_eval_author_handoff( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: @@ -35,6 +35,13 @@ async def test_insight_run_stages_inputs_before_eval_author( experiment = tmp_path / "experiment" captured: dict[str, Path] = {} + class EvalAuthorHandoff: + def __init__(self, train_dataset: SimpleNamespace, validation_dataset: SimpleNamespace) -> None: + self.train_dataset = train_dataset + self.validation_dataset = validation_dataset + self.insight_suite = None + self.metric_keys = ("reward",) + class RecordingDatasetFactory: def build_dataset(self, evaluator_type: str, ref: DatasetRef) -> SimpleNamespace: return SimpleNamespace(ref=ref) @@ -53,13 +60,13 @@ async def run( train_dataset: SimpleNamespace, validation_dataset: SimpleNamespace, **kwargs: object, - ) -> None: + ) -> EvalAuthorHandoff: captured["train"] = Path(train_dataset.ref.uri) captured["validation"] = Path(validation_dataset.ref.uri) captured["template"] = Path(task_template.uri) (captured["train"] / "task-1" / "tests" / "test.sh").write_text("curated", encoding="utf-8") (captured["template"].parent / "generated-task").mkdir() - raise RuntimeError("stop after eval_author") + return EvalAuthorHandoff(train_dataset, validation_dataset) monkeypatch.setattr( loop_module, @@ -73,6 +80,12 @@ async def run( "_init_structure", lambda self: (experiment / "agents", experiment / "analysis", experiment / "results"), ) + monkeypatch.setattr(EvolutionaryOptimizer, "_detect_last_round", lambda self: None) + monkeypatch.setattr( + EvolutionaryOptimizer, + "_create_experiment_run", + AsyncMock(side_effect=RuntimeError("stop after eval_author")), + ) backend = SimpleNamespace( client=object(), diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_loop_helpers.py b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_helpers.py new file mode 100644 index 0000000000..4ddb85007f --- /dev/null +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_helpers.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json +from pathlib import Path + +from nemo_experimentalist_plugin.experimentalist.components.loop import EvolutionaryOptimizer + + +def test_rollback_removes_all_result_directories_for_removed_candidate(tmp_path: Path) -> None: + root = tmp_path / "eval-and-optimize" + agents_dir = root / "agents" + results_dir = root / "results" + analysis_dir = root / "analysis" + smoke_dataset_dir = root / "smoke-dataset" + smoke_results_dir = root / "smoke-results" + for directory in (agents_dir, results_dir, analysis_dir, smoke_dataset_dir, smoke_results_dir): + directory.mkdir(parents=True) + + removed_agent = agents_dir / "agent-2" + removed_agent.mkdir() + (removed_agent / "metadata.json").write_text(json.dumps({"round": 2})) + surviving_agent = agents_dir / "agent-20" + surviving_agent.mkdir() + (surviving_agent / "metadata.json").write_text(json.dumps({"round": 1})) + + removed_results = [ + results_dir / "agent-2-train", + results_dir / "agent-2-validation", + results_dir / "agent-2-custom-channel", + ] + for result_dir in removed_results: + result_dir.mkdir() + surviving_result = results_dir / "agent-20-custom-channel" + surviving_result.mkdir() + + optimizer = object.__new__(EvolutionaryOptimizer) + optimizer.working_dir = tmp_path + optimizer._delete_all_artifacts(from_round=1) + + assert not removed_agent.exists() + assert all(not result_dir.exists() for result_dir in removed_results) + assert surviving_agent.is_dir() + assert surviving_result.is_dir() diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_loop_insight_suite.py b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_insight_suite.py deleted file mode 100644 index 75768c4aa4..0000000000 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_loop_insight_suite.py +++ /dev/null @@ -1,505 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -from pathlib import Path -from types import SimpleNamespace -from typing import Any -from unittest.mock import AsyncMock - -import pytest -from nemo_experimentalist_plugin.config import EvolutionaryOptimizerConfig -from nemo_experimentalist_plugin.entities import ( - Candidate, - Dataset, - DatasetRef, - DataValue, - EvaluationResult, - MetricResult, - ResourceRef, - RewardRecord, - Task, - TrialResult, -) -from nemo_experimentalist_plugin.experimentalist.components import loop as loop_module -from nemo_experimentalist_plugin.experimentalist.components.evaluator.harbor import HarborEvaluatorConfig -from nemo_experimentalist_plugin.experimentalist.components.loop import EvolutionaryOptimizer - - -class _StopAfterOneRound(Exception): - pass - - -def _suite_metadata(identity_char: str = "a") -> dict[str, DataValue]: - identity = f"sha256:{identity_char * 64}" - return { - "insight_suite_identity": identity, - "insight_suite_scorer_identity": f"sha256:{'b' * 64}", - "insight_suite_task_hashes": { - "insight-task": { - "content_hash": f"sha256:{'c' * 64}", - "verifier_hash": f"sha256:{'d' * 64}", - } - }, - } - - -def _insight_result(label: str, score: float) -> EvaluationResult: - return EvaluationResult( - id=f"{label}-insight", - aggregate_metrics={"uses_required_tool": score}, - trials=[ - TrialResult( - id=f"{label}-insight-task-1", - task_id="insight-task", - attempt=1, - status="completed", - metrics={ - "uses_required_tool": MetricResult( - name="uses_required_tool", - value=score, - ) - }, - ) - ], - ) - - -@pytest.mark.asyncio -async def test_insight_run_evaluates_and_persists_baseline_and_new_candidate_metrics( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - train_dataset = Dataset(id="train") - validation_dataset = Dataset(id="validation") - insight_dataset = Dataset( - id="insight-suite", - source=ResourceRef(uri="file:///experiment/eval-and-optimize/eval_author/insight-1/insight-suite"), - tasks=[Task(id="insight-task")], - metadata=_suite_metadata(), - ) - datasets = { - "train": train_dataset, - "validation": validation_dataset, - } - - class RecordingDatasetFactory: - def build_dataset(self, evaluator_type: str, ref: DatasetRef) -> Dataset: - return datasets[ref.uri] - - def build_task_template(self, evaluator_type: str, ref: DatasetRef) -> Task: - return Task(id="template", uri=ref.uri) - - class ReturningEvalAuthor: - def __init__(self, **kwargs: object) -> None: - pass - - async def run(self, **kwargs: Any) -> SimpleNamespace: - return SimpleNamespace( - train_dataset=kwargs["train_dataset"], - validation_dataset=kwargs["validation_dataset"], - insight_suite=insight_dataset, - ) - - baseline = Candidate(run_id="run-1", label="agent-0", round=0, optimization="baseline") - new_candidate = Candidate( - run_id="run-1", - label="agent-1", - ancestor="agent-0", - round=1, - optimization="use the required tool", - ) - insight_results = { - "agent-0": _insight_result("agent-0", 0.0), - "agent-1": _insight_result("agent-1", 1.0), - } - insight_evaluations: list[tuple[Dataset, list[Candidate]]] = [] - - async def evaluate_insight_candidates( - self: EvolutionaryOptimizer, - *, - dataset: Dataset, - evaluator: object, - candidates: list[Candidate], - ) -> dict[str, EvaluationResult]: - insight_evaluations.append((dataset, candidates)) - return {candidate.label: insight_results[candidate.label] for candidate in candidates} - - async def evaluate_validation_candidates( - self: EvolutionaryOptimizer, - *, - candidates: list[Candidate], - **kwargs: object, - ) -> dict[str, EvaluationResult]: - return { - candidate.label: EvaluationResult( - id=f"{candidate.label}-validation", - aggregate_metrics={"reward": 0.5}, - ) - for candidate in candidates - if "validation" not in candidate.rewards - } - - async def update_candidate( - self: EvolutionaryOptimizer, - candidate: Candidate, - *, - updates: dict[str, object] | None = None, - **kwargs: object, - ) -> None: - for key, value in (updates or {}).items(): - setattr(candidate, key, value) - - run_entity = SimpleNamespace(id="run-1", status="running", rounds_completed=0) - backend = SimpleNamespace( - client=object(), - get_insight=AsyncMock(return_value=SimpleNamespace(agent="agent-source")), - get_agent_code=AsyncMock(), - persist_evaluation=AsyncMock(), - update_run=AsyncMock(), - ) - evolution_tree = SimpleNamespace(survivors=lambda round_num: [baseline], add=lambda candidate: None) - - class StopAfterOneRoundTerminator: - calls = 0 - - async def run(self, **kwargs: object) -> SimpleNamespace: - self.calls += 1 - if self.calls > 1: - raise _StopAfterOneRound - return SimpleNamespace(stop=False, reason="continue") - - monkeypatch.setattr(loop_module, "DatasetFactory", RecordingDatasetFactory) - monkeypatch.setattr( - loop_module, - "EvaluatorFactory", - lambda: SimpleNamespace(build_evaluator=lambda *args, **kwargs: object()), - ) - monkeypatch.setattr(loop_module, "EvalAuthor", ReturningEvalAuthor) - monkeypatch.setattr( - loop_module, - "stage_eval_author_inputs", - AsyncMock(side_effect=lambda _, **refs: SimpleNamespace(**refs)), - ) - monkeypatch.setattr(loop_module.EvolutionTree, "from_dir", lambda path: evolution_tree) - monkeypatch.setattr(EvolutionaryOptimizer, "_detect_last_round", lambda self: None) - monkeypatch.setattr(EvolutionaryOptimizer, "_create_experiment_run", AsyncMock(return_value=run_entity)) - monkeypatch.setattr(EvolutionaryOptimizer, "_create_baseline_agent", AsyncMock(return_value=baseline)) - monkeypatch.setattr(EvolutionaryOptimizer, "_update_candidate", update_candidate) - monkeypatch.setattr( - EvolutionaryOptimizer, - "_evaluate_validation_candidates", - evaluate_validation_candidates, - ) - monkeypatch.setattr( - EvolutionaryOptimizer, - "_evaluate_insight_candidates", - evaluate_insight_candidates, - raising=False, - ) - monkeypatch.setattr(EvolutionaryOptimizer, "_generate_initial_goal_tree", AsyncMock()) - monkeypatch.setattr( - EvolutionaryOptimizer, - "_evaluate_train_candidates", - AsyncMock( - return_value={ - "agent-0": EvaluationResult(id="agent-0-train", aggregate_metrics={"reward": 0.5}), - } - ), - ) - monkeypatch.setattr(EvolutionaryOptimizer, "_analyze_round", AsyncMock(return_value="round analysis")) - monkeypatch.setattr(EvolutionaryOptimizer, "_update_goal_tree", AsyncMock()) - monkeypatch.setattr(EvolutionaryOptimizer, "_propose_improvements", AsyncMock(return_value=[object()])) - monkeypatch.setattr(EvolutionaryOptimizer, "_create_agent", lambda self, **kwargs: new_candidate) - monkeypatch.setattr( - EvolutionaryOptimizer, - "_implement_candidates", - AsyncMock(side_effect=lambda **kwargs: kwargs["candidates"]), - ) - - config = EvolutionaryOptimizerConfig(disable_trajectory_scoring=True) - optimizer = object.__new__(EvolutionaryOptimizer) - optimizer.working_dir = tmp_path - optimizer.config = config - optimizer.shell = SimpleNamespace(close=AsyncMock()) - optimizer.terminator = StopAfterOneRoundTerminator() - deps = SimpleNamespace( - backend=backend, - workspace="default", - config=config, - evaluator_type="harbor", - train_dataset=DatasetRef(uri="train"), - validation_dataset=DatasetRef(uri="validation"), - task_template=DatasetRef(uri="template"), - insight="insight-1", - agent=None, - agent_spec=None, - ) - - with pytest.raises(_StopAfterOneRound): - await optimizer.run(deps) - - assert insight_evaluations == [ - (insight_dataset, [baseline]), - (insight_dataset, [new_candidate]), - ] - assert baseline.reward("insight").metrics == {"uses_required_tool": 0.0} - assert new_candidate.reward("insight").metrics == {"uses_required_tool": 1.0} - assert baseline.rewards["insight"].metadata["suite_identity"] == f"sha256:{'a' * 64}" - assert new_candidate.rewards["insight"].metadata["suite_identity"] == f"sha256:{'a' * 64}" - assert baseline.rewards["insight"].metadata["metric_keys"] == ["uses_required_tool"] - insight_persistence = [ - call.kwargs for call in backend.persist_evaluation.await_args_list if call.kwargs["split"] == "insight" - ] - assert [call["candidate"] for call in insight_persistence] == [baseline, new_candidate] - assert [call["result"].id for call in insight_persistence] == [ - insight_results["agent-0"].id, - insight_results["agent-1"].id, - ] - assert all( - call["result"].metadata["insight_suite_identity"] == f"sha256:{'a' * 64}" for call in insight_persistence - ) - assert all( - trial.metadata["insight_suite_scorer_identity"] == f"sha256:{'b' * 64}" - for call in insight_persistence - for trial in call["result"].trials - ) - - -@pytest.mark.asyncio -async def test_insight_evaluation_skips_cached_candidates_and_empty_suites( - monkeypatch: pytest.MonkeyPatch, -) -> None: - cached = Candidate( - run_id="run-1", - label="agent-0", - round=0, - optimization="baseline", - rewards={ - "insight": RewardRecord( - metrics={"uses_required_tool": 0.0}, - trials=[], - metadata={"suite_identity": f"sha256:{'a' * 64}", "metric_keys": ["uses_required_tool"]}, - ) - }, - ) - pending = Candidate( - run_id="run-1", - label="agent-1", - round=1, - optimization="use the required tool", - ) - result = _insight_result("agent-1", 1.0) - evaluate_agent = AsyncMock(return_value=(pending, result)) - monkeypatch.setattr(EvolutionaryOptimizer, "_evaluate_agent", evaluate_agent) - optimizer = object.__new__(EvolutionaryOptimizer) - - evaluated = await optimizer._evaluate_insight_candidates( - dataset=Dataset( - id="insight-suite", - source=ResourceRef(uri="file:///experiment/eval-and-optimize/eval_author/insight-1/insight-suite"), - tasks=[Task(id="insight-task")], - metadata=_suite_metadata(), - ), - evaluator=object(), # type: ignore[arg-type] - candidates=[cached, pending], - ) - - assert evaluated == {"agent-1": result} - assert evaluate_agent.await_args is not None - assert evaluate_agent.await_args.args[0] is pending - assert evaluate_agent.await_args.kwargs["minimum_attempts"] == 2 - - empty = await optimizer._evaluate_insight_candidates( - dataset=Dataset(id="empty-insight-suite"), - evaluator=object(), # type: ignore[arg-type] - candidates=[pending], - ) - assert empty == {} - assert evaluate_agent.await_count == 1 - - -@pytest.mark.asyncio -async def test_insight_evaluation_reuses_only_matching_suite_identity( - monkeypatch: pytest.MonkeyPatch, -) -> None: - cached = Candidate( - run_id="run-1", - label="agent-0", - round=0, - optimization="baseline", - rewards={ - "insight": RewardRecord( - metrics={"uses_required_tool": 0.0}, - trials=[], - metadata={"suite_identity": f"sha256:{'a' * 64}", "metric_keys": ["uses_required_tool"]}, - ) - }, - ) - result = _insight_result("agent-0", 0.5) - evaluate_agent = AsyncMock(return_value=(cached, result)) - monkeypatch.setattr(EvolutionaryOptimizer, "_evaluate_agent", evaluate_agent) - optimizer = object.__new__(EvolutionaryOptimizer) - - matching = Dataset( - id="insight-suite", - source=ResourceRef(uri="file:///experiment/eval-and-optimize/eval_author/insight-1/insight-suite"), - tasks=[Task(id="insight-task")], - metadata=_suite_metadata("a"), - ) - changed = Dataset( - id="insight-suite", - source=ResourceRef(uri="file:///experiment/eval-and-optimize/eval_author/insight-1/insight-suite"), - tasks=[Task(id="insight-task")], - metadata=_suite_metadata("e"), - ) - - assert ( - await optimizer._evaluate_insight_candidates( - dataset=matching, - evaluator=object(), # type: ignore[arg-type] - candidates=[cached], - ) - == {} - ) - assert await optimizer._evaluate_insight_candidates( - dataset=changed, - evaluator=object(), # type: ignore[arg-type] - candidates=[cached], - ) == {"agent-0": result} - evaluate_agent.assert_awaited_once() - - -@pytest.mark.asyncio -async def test_cached_insight_metric_keys_are_order_independent( - monkeypatch: pytest.MonkeyPatch, -) -> None: - identity = f"sha256:{'a' * 64}" - candidates = [ - Candidate( - run_id="run-1", - label="agent-0", - round=0, - optimization="baseline", - rewards={ - "insight": RewardRecord( - metrics={"reward": 0.5, "uses_required_tool": 0.0}, - trials=[], - metadata={"suite_identity": identity, "metric_keys": ["uses_required_tool", "reward"]}, - ) - }, - ), - Candidate( - run_id="run-1", - label="agent-1", - round=1, - optimization="improve tool use", - rewards={ - "insight": RewardRecord( - metrics={"reward": 0.75, "uses_required_tool": 1.0}, - trials=[], - metadata={"suite_identity": identity, "metric_keys": ["reward", "uses_required_tool"]}, - ) - }, - ), - ] - dataset = Dataset( - id="insight-suite", - source=ResourceRef(uri="file:///experiment/eval-and-optimize/eval_author/insight-1/insight-suite"), - tasks=[Task(id="insight-task")], - metadata={ - **_suite_metadata(), - "insight_metric_keys": ["uses_required_tool", "reward"], - }, - ) - optimizer = object.__new__(EvolutionaryOptimizer) - monkeypatch.setattr( - optimizer, - "_evaluate_insight_candidates", - AsyncMock(return_value={}), - ) - - await optimizer._evaluate_and_persist_insight_candidates( - dataset=dataset, - evaluator=object(), # type: ignore[arg-type] - candidates=candidates, - workspace="default", - backend=SimpleNamespace(), - run_id="run-1", - ) - - assert dataset.metadata["insight_metric_keys"] == ["reward", "uses_required_tool"] - - -@pytest.mark.asyncio -async def test_insight_evaluation_uses_at_least_two_attempts_without_changing_other_splits( - tmp_path: Path, -) -> None: - candidate = Candidate( - run_id="run-1", - label="agent-0", - round=0, - optimization="baseline", - ) - received_attempts: list[int] = [] - - class RecordingEvaluator: - options = HarborEvaluatorConfig(n_attempts=1) - - async def run(self, **kwargs: object) -> EvaluationResult: - options = kwargs["options"] - assert isinstance(options, HarborEvaluatorConfig) - received_attempts.append(options.n_attempts) - return _insight_result(candidate.label, 0.5) - - optimizer = object.__new__(EvolutionaryOptimizer) - optimizer.working_dir = tmp_path - evaluator = RecordingEvaluator() - dataset = Dataset(id="insight-suite") - - await optimizer._evaluate_agent(candidate, dataset, evaluator) # type: ignore[arg-type] - await optimizer._evaluate_agent( - candidate, - dataset, - evaluator, # type: ignore[arg-type] - minimum_attempts=2, - ) - - assert received_attempts == [1, 2] - assert evaluator.options.n_attempts == 1 - - -def test_rollback_removes_digest_named_insight_results(tmp_path: Path) -> None: - root = tmp_path / "eval-and-optimize" - agents_dir = root / "agents" - results_dir = root / "results" - analysis_dir = root / "analysis" - smoke_dataset_dir = root / "smoke-dataset" - smoke_results_dir = root / "smoke-results" - for directory in (agents_dir, results_dir, analysis_dir, smoke_dataset_dir, smoke_results_dir): - directory.mkdir(parents=True) - - removed_agent = agents_dir / "agent-2" - removed_agent.mkdir() - (removed_agent / "metadata.json").write_text('{"round": 2}\n') - surviving_agent = agents_dir / "agent-20" - surviving_agent.mkdir() - (surviving_agent / "metadata.json").write_text('{"round": 1}\n') - - removed_results = [ - results_dir / "agent-2-train", - results_dir / "agent-2-validation", - results_dir / "agent-2-insight-abcdef123456", - ] - for result_dir in removed_results: - result_dir.mkdir() - surviving_result = results_dir / "agent-20-insight-abcdef123456" - surviving_result.mkdir() - - optimizer = object.__new__(EvolutionaryOptimizer) - optimizer.working_dir = tmp_path - optimizer._delete_all_artifacts(from_round=1) - - assert not removed_agent.exists() - assert all(not result_dir.exists() for result_dir in removed_results) - assert surviving_agent.is_dir() - assert surviving_result.is_dir() diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_loop_reporting.py b/plugins/nemo-experimentalist/tests/experimentalist/test_loop_reporting.py deleted file mode 100644 index a019d4943a..0000000000 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_loop_reporting.py +++ /dev/null @@ -1,607 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -import math -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import AsyncMock - -import pytest -from nemo_experimentalist_plugin.entities import ( - Candidate, - Dataset, - EvaluationResult, - MetricResult, - ResourceRef, - RewardRecord, - Task, - TrialResult, -) -from nemo_experimentalist_plugin.experimentalist.components.evaluator import TrialStatus -from nemo_experimentalist_plugin.experimentalist.components.insight_promotion import ( - _task_evidence, - insight_suite_provenance, - render_insight_promotion_section, - select_insight_promotion_suggestions, - validate_insight_evaluation_result, - write_insight_comparison_section, - write_insight_promotion_section, -) -from nemo_experimentalist_plugin.experimentalist.components.loop import AnalysisSkill, EvolutionaryOptimizer -from nemo_experimentalist_plugin.experimentalist.components.models import EvolutionTree - -_SUITE_IDENTITY = f"sha256:{'a' * 64}" -_SUITE_PATH = Path("/experiment/eval-and-optimize/eval_author/insight-1/insight-suite") - - -def _candidate( - label: str, - *, - round_num: int, - rewards: dict[str, RewardRecord] | None = None, -) -> Candidate: - channels = dict(rewards or {}) - insight = channels.get("insight") or RewardRecord() - channels["insight"] = insight.model_copy( - update={"metadata": {"suite_identity": _SUITE_IDENTITY, "metric_keys": ["reward", "uses_required_tool"]}} - ) - return Candidate( - run_id="run-1", - label=label, - round=round_num, - optimization="baseline" if round_num == 0 else "improve required tool use", - rewards=channels, - ) - - -def _insight_dataset(tasks: list[Task]) -> Dataset: - local_tasks = [ - task if task.uri else task.model_copy(update={"uri": (_SUITE_PATH / task.id).as_uri()}) for task in tasks - ] - return Dataset( - id="insight", - source=ResourceRef(uri=_SUITE_PATH.as_uri()), - tasks=local_tasks, - metadata={ - "insight_suite_identity": _SUITE_IDENTITY, - "insight_suite_scorer_identity": f"sha256:{'b' * 64}", - "insight_suite_task_hashes": { - task.id: { - "content_hash": f"sha256:{'c' * 64}", - "verifier_hash": f"sha256:{'d' * 64}", - } - for task in local_tasks - }, - }, - ) - - -def test_round_analysis_contract_requires_separate_insight_suite_dimensions() -> None: - skill_prompt = " ".join((AnalysisSkill.__doc__ or "").split()) - merge_prompt = " ".join((EvolutionaryOptimizer.merge_analysis.__doc__ or "").split()) - - assert "Insight Suite Reward" in skill_prompt - assert "candidate.rewards" in skill_prompt - assert "separate from train and validation rewards" in skill_prompt - assert "insight_dim_keys" in merge_prompt - assert "candidate.round == 0" in merge_prompt - assert "must name every available Insight Suite dimension" in merge_prompt - assert "Never blend those metrics into train/validation rewards" in merge_prompt - assert "adaptive/development feedback" in merge_prompt - assert "never present them as independent validation evidence" in merge_prompt - - -def test_final_report_contract_requires_baseline_winner_insight_comparison() -> None: - report_prompt = " ".join((EvolutionaryOptimizer.write_final_report.__doc__ or "").split()) - - assert "Insight Suite Metrics table" in report_prompt - assert "baseline, winner, and signed delta columns" in report_prompt - assert "Keep this table separate from generic train and validation rewards" in report_prompt - - -def test_terminal_summary_includes_baseline_and_winner_insight_metrics() -> None: - baseline = _candidate( - "agent-0", - round_num=0, - rewards={"insight": RewardRecord(metrics={"uses_required_tool": 0.0})}, - ) - winner = _candidate( - "agent-1", - round_num=1, - rewards={ - "insight": RewardRecord(metrics={"uses_required_tool": 1.0}), - "validation": RewardRecord(metrics={"reward": 0.75}), - }, - ) - optimizer = object.__new__(EvolutionaryOptimizer) - - summary = optimizer._render_summary(rounds_completed=1, baseline=baseline, winner=winner) - - assert "validation_reward={'reward': 0.75}" in summary - assert "insight_suite=(baseline={'uses_required_tool': 0.0}" in summary - assert "winner={'uses_required_tool': 1.0})" in summary - - -def test_terminal_summary_omits_insight_comparison_when_unavailable() -> None: - baseline = _candidate("agent-0", round_num=0) - winner = _candidate("agent-1", round_num=1, rewards={"validation": RewardRecord(metrics={"reward": 0.75})}) - optimizer = object.__new__(EvolutionaryOptimizer) - - summary = optimizer._render_summary(rounds_completed=1, baseline=baseline, winner=winner) - - assert "insight_suite" not in summary - - -def _insight_trial( - task_id: str, - score: float, - *, - attempt: int = 1, - status: TrialStatus = "completed", -) -> TrialResult: - return TrialResult( - id=f"{task_id}-{attempt}", - task_id=task_id, - attempt=attempt, - status=status, - metrics={ - "reward": MetricResult(name="reward", value=1.0), - "uses_required_tool": MetricResult(name="uses_required_tool", value=score), - }, - ) - - -def test_insight_promotion_suggestions_are_stable_discriminative_and_diverse( - tmp_path: Path, -) -> None: - tasks = [ - Task(id="task-a", uri=(tmp_path / "task-a").as_uri()), - Task(id="task-b", uri=(tmp_path / "task-b").as_uri()), - Task(id="task-c", uri=(tmp_path / "task-c").as_uri()), - Task(id="task-flaky", uri=(tmp_path / "task-flaky").as_uri()), - Task(id="task-flat", uri=(tmp_path / "task-flat").as_uri()), - ] - baseline = _candidate("agent-0", round_num=0) - baseline.record_reward( - "insight", - trials=[ - _insight_trial("task-a", 0.0, attempt=1), - _insight_trial("task-a", 0.0, attempt=2), - _insight_trial("task-b", 0.0, attempt=1), - _insight_trial("task-b", 0.0, attempt=2), - _insight_trial("task-c", 0.8, attempt=1), - _insight_trial("task-c", 0.8, attempt=2), - _insight_trial("task-flaky", 0.0, attempt=1), - _insight_trial("task-flaky", 1.0, attempt=2), - _insight_trial("task-flat", 0.5, attempt=1), - _insight_trial("task-flat", 0.5, attempt=2), - ], - ) - winner = _candidate("agent-1", round_num=1) - winner.record_reward( - "insight", metadata={**winner.rewards["insight"].metadata, "metric_keys": ["uses_required_tool", "reward"]} - ) - winner.record_reward( - "insight", - trials=[ - _insight_trial("task-a", 1.0, attempt=1), - _insight_trial("task-a", 1.0, attempt=2), - _insight_trial("task-b", 1.0, attempt=1), - _insight_trial("task-b", 1.0, attempt=2), - _insight_trial("task-c", 0.9, attempt=1), - _insight_trial("task-c", 0.9, attempt=2), - _insight_trial("task-flaky", 1.0, attempt=1), - _insight_trial("task-flaky", 1.0, attempt=2), - _insight_trial("task-flat", 0.5, attempt=1), - _insight_trial("task-flat", 0.5, attempt=2), - ], - ) - - suggestions = select_insight_promotion_suggestions( - _insight_dataset(tasks), - [baseline, winner], - winner=winner, - ) - - assert [suggestion.task_id for suggestion in suggestions] == ["task-a", "task-c"] - assert suggestions[0].metric_name == "uses_required_tool" - assert suggestions[0].discrimination == 1.0 - assert suggestions[0].diversity_score is None - assert suggestions[1].diversity_score == pytest.approx(0.45) - - section = render_insight_promotion_section(suggestions) - assert "Advisory adaptive/development evidence only" in section - assert "`task-a`" in section - assert str(tmp_path / "task-a") in section - assert _SUITE_IDENTITY in section - assert "baseline 0.00 → winner 1.00" in section - assert f"task sha256:{'c' * 64}" in section - assert "task-b" not in section - assert "task-flaky" not in section - assert "task-flat" not in section - - -def test_task_evidence_excludes_candidates_from_other_suites(tmp_path: Path) -> None: - dataset = _insight_dataset([Task(id="task-a", uri=(tmp_path / "task-a").as_uri())]) - task = dataset.list_tasks()[0] - baseline = _candidate("agent-0", round_num=0) - winner = _candidate("agent-1", round_num=1) - stale = _candidate("agent-stale", round_num=1) - stale.record_reward( - "insight", metadata={**stale.rewards["insight"].metadata, "suite_identity": f"sha256:{'e' * 64}"} - ) - for candidate, score in ((baseline, 0.0), (winner, 1.0), (stale, 0.5)): - candidate.record_reward( - "insight", - trials=[ - _insight_trial(task.id, score, attempt=1), - _insight_trial(task.id, score, attempt=2), - ], - ) - - evidence = _task_evidence( - task, - [baseline, winner, stale], - baseline=baseline, - winner=winner, - provenance=insight_suite_provenance(dataset), - ) - - assert evidence is not None - assert evidence.suggestion.candidate_count == 2 - assert evidence.suggestion.total_attempts == 4 - assert set(candidate_label for candidate_label, _ in evidence.profile) == { - baseline.label, - winner.label, - } - - -def test_insight_promotion_section_explains_when_no_task_qualifies() -> None: - section = render_insight_promotion_section([]) - - assert "## Insight Suite Promotion Suggestions" in section - assert "No task had complete repeated evidence" in section - - -def test_insight_promotion_section_is_appended_without_rewriting_report( - tmp_path: Path, -) -> None: - report_path = tmp_path / "eval-and-optimize" / "OPTIMIZATION.md" - report_path.parent.mkdir(parents=True) - report_path.write_text("# Optimization\n\nExisting analysis.\n") - - write_insight_promotion_section(report_path, []) - first_report = report_path.read_text() - write_insight_promotion_section(report_path, []) - - assert report_path.read_text() == first_report - assert first_report.startswith("# Optimization\n\nExisting analysis.") - assert first_report.count("## Insight Suite Promotion Suggestions") == 1 - - -@pytest.mark.parametrize("score", [1.1, -0.1, math.inf, -math.inf, math.nan]) -def test_runtime_insight_metrics_reject_out_of_range_and_non_finite_values(score: float) -> None: - result = EvaluationResult( - id="invalid", - aggregate_metrics={"reward": 1.0, "uses_required_tool": score}, - trials=[_insight_trial("task-a", score)], - ) - - with pytest.raises(ValueError, match=r"finite and within \[0, 1\]"): - validate_insight_evaluation_result(result) - - -def test_runtime_insight_metrics_reject_missing_or_inconsistent_keys() -> None: - missing_trial_key = EvaluationResult( - id="missing", - aggregate_metrics={"reward": 1.0, "uses_required_tool": 0.5}, - trials=[ - TrialResult( - id="task-a-1", - task_id="task-a", - status="completed", - metrics={"reward": MetricResult(name="reward", value=1.0)}, - ) - ], - ) - - with pytest.raises(ValueError, match="metric keys are inconsistent"): - validate_insight_evaluation_result(missing_trial_key) - - with pytest.raises(ValueError, match="aggregate metric keys are inconsistent"): - validate_insight_evaluation_result( - EvaluationResult( - id="changed", - aggregate_metrics={"reward": 1.0, "different_metric": 0.5}, - trials=[ - TrialResult( - id="task-a-1", - task_id="task-a", - status="completed", - metrics={ - "reward": MetricResult(name="reward", value=1.0), - "different_metric": MetricResult(name="different_metric", value=0.5), - }, - ) - ], - ), - expected_metric_keys=["reward", "uses_required_tool"], - ) - - -@pytest.mark.parametrize( - ("invalid_score", "missing_key"), - [(1.1, False), (math.nan, False), (0.5, True)], -) -def test_invalid_runtime_metrics_cannot_be_promotion_evidence( - invalid_score: float, - missing_key: bool, -) -> None: - task = Task(id="task-a") - baseline = _candidate("agent-0", round_num=0) - winner = _candidate("agent-1", round_num=1) - baseline.record_reward( - "insight", - trials=[ - _insight_trial("task-a", 0.0, attempt=1), - _insight_trial("task-a", 0.0, attempt=2), - ], - ) - invalid_trial = _insight_trial("task-a", invalid_score, attempt=1) - if missing_key: - invalid_trial.metrics.pop("uses_required_tool") - winner.record_reward( - "insight", - trials=[ - invalid_trial, - _insight_trial("task-a", 1.0, attempt=2), - ], - ) - - assert ( - select_insight_promotion_suggestions( - _insight_dataset([task]), - [baseline, winner], - winner=winner, - ) - == [] - ) - - -def test_one_attempt_failed_and_incomplete_evidence_do_not_qualify_as_stable() -> None: - task = Task(id="task-a") - baseline = _candidate("agent-0", round_num=0) - winner = _candidate("agent-1", round_num=1) - baseline.record_reward("insight", trials=[_insight_trial("task-a", 0.0)]) - winner.record_reward("insight", trials=[_insight_trial("task-a", 1.0)]) - - assert ( - select_insight_promotion_suggestions( - _insight_dataset([task]), - [baseline, winner], - winner=winner, - ) - == [] - ) - - baseline.reward("insight").trials.append(_insight_trial("task-a", 0.0, attempt=2)) - winner.reward("insight").trials.append(_insight_trial("task-a", 1.0, attempt=2, status="failed")) - assert ( - select_insight_promotion_suggestions( - _insight_dataset([task]), - [baseline, winner], - winner=winner, - ) - == [] - ) - - winner.record_reward("insight", trials=[]) - assert ( - select_insight_promotion_suggestions( - _insight_dataset([task]), - [baseline, winner], - winner=winner, - ) - == [] - ) - - -@pytest.mark.parametrize( - ("baseline_score", "winner_score", "bad_score"), - [ - (0.5, 0.5, None), - (0.8, 0.2, None), - (0.5, 0.5, 0.0), - ], -) -def test_promotion_requires_baseline_to_winner_improvement( - baseline_score: float, - winner_score: float, - bad_score: float | None, -) -> None: - task = Task(id="task-a") - baseline = _candidate("agent-0", round_num=0) - winner = _candidate("agent-1", round_num=1) - candidates = [baseline, winner] - baseline.record_reward( - "insight", - trials=[ - _insight_trial("task-a", baseline_score, attempt=1), - _insight_trial("task-a", baseline_score, attempt=2), - ], - ) - winner.record_reward( - "insight", - trials=[ - _insight_trial("task-a", winner_score, attempt=1), - _insight_trial("task-a", winner_score, attempt=2), - ], - ) - if bad_score is not None: - bad = _candidate("agent-bad", round_num=1) - bad.record_reward( - "insight", - trials=[ - _insight_trial("task-a", bad_score, attempt=1), - _insight_trial("task-a", bad_score, attempt=2), - ], - ) - candidates.append(bad) - - assert ( - select_insight_promotion_suggestions( - _insight_dataset([task]), - candidates, - winner=winner, - ) - == [] - ) - - -def test_deterministic_insight_comparison_section_uses_local_suite_identity( - tmp_path: Path, -) -> None: - baseline = _candidate( - "agent-0", - round_num=0, - rewards={"insight": RewardRecord(metrics={"reward": 0.5, "uses_required_tool": 0.0})}, - ) - winner = _candidate( - "agent-1", - round_num=1, - rewards={"insight": RewardRecord(metrics={"reward": 0.75, "uses_required_tool": 1.0})}, - ) - report_path = tmp_path / "OPTIMIZATION.md" - provenance = insight_suite_provenance(_insight_dataset([Task(id="task-a")])) - - write_insight_comparison_section(report_path, baseline, winner, provenance) - report = report_path.read_text() - - assert "## Deterministic Insight Suite Comparison" in report - assert str(_SUITE_PATH) in report - assert _SUITE_IDENTITY in report - assert "| `uses_required_tool` | 0.000 | 1.000 | +1.000 |" in report - - -@pytest.mark.asyncio -async def test_final_report_failure_preserves_compact_summary_and_deterministic_sections( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - baseline = _candidate( - "agent-0", - round_num=0, - rewards={ - "insight": RewardRecord(metrics={"reward": 0.5, "uses_required_tool": 0.0}), - "validation": RewardRecord(metrics={"reward": 0.5}), - }, - ) - winner = _candidate( - "agent-1", - round_num=1, - rewards={ - "insight": RewardRecord(metrics={"reward": 0.75, "uses_required_tool": 1.0}), - "validation": RewardRecord(metrics={"reward": 0.75}), - }, - ) - tree = EvolutionTree() - tree.add(baseline) - tree.add(winner) - optimizer = object.__new__(EvolutionaryOptimizer) - optimizer.working_dir = tmp_path - (tmp_path / "eval-and-optimize").mkdir() - monkeypatch.setattr(optimizer, "_copy_best_to_workspace", lambda best_id: None) - original_report_writer = EvolutionaryOptimizer.write_final_report - type.__setattr__( - EvolutionaryOptimizer, - "write_final_report", - AsyncMock(side_effect=RuntimeError("LLM report failed")), - ) - run = SimpleNamespace(status="running", winner_agent=None, rounds_completed=1) - backend = SimpleNamespace(update_run=AsyncMock()) - - try: - finalized = await optimizer._finalize( - workspace="default", - backend=backend, - agents_dir=tmp_path / "eval-and-optimize" / "agents", - run_entity=run, - evolution_tree=tree, - agent_name="agent", - insight_dataset=_insight_dataset([Task(id="task-a")]), - ) - finally: - type.__setattr__( - EvolutionaryOptimizer, - "write_final_report", - original_report_writer, - ) - - report = (tmp_path / "eval-and-optimize" / "OPTIMIZATION.md").read_text() - assert finalized is winner - assert "## Compact Run Summary" in report - assert "Optimization complete: 1 round(s) completed" in report - assert "## Deterministic Insight Suite Comparison" in report - assert "## Insight Suite Promotion Suggestions" in report - - -@pytest.mark.asyncio -async def test_insight_report_mismatch_does_not_fail_completed_run( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - caplog: pytest.LogCaptureFixture, -) -> None: - baseline = _candidate( - "agent-0", - round_num=0, - rewards={ - "insight": RewardRecord(metrics={"reward": 0.5, "uses_required_tool": 0.0}), - "validation": RewardRecord(metrics={"reward": 0.5}), - }, - ) - winner = _candidate( - "agent-1", - round_num=1, - rewards={ - "insight": RewardRecord(metrics={"reward": 0.75, "uses_required_tool": 1.0}), - "validation": RewardRecord(metrics={"reward": 0.75}), - }, - ) - winner.record_reward( - "insight", metadata={**winner.rewards["insight"].metadata, "suite_identity": f"sha256:{'e' * 64}"} - ) - tree = EvolutionTree() - tree.add(baseline) - tree.add(winner) - optimizer = object.__new__(EvolutionaryOptimizer) - optimizer.working_dir = tmp_path - (tmp_path / "eval-and-optimize").mkdir() - monkeypatch.setattr(optimizer, "_copy_best_to_workspace", lambda best_id: None) - original_report_writer = EvolutionaryOptimizer.write_final_report - type.__setattr__(EvolutionaryOptimizer, "write_final_report", AsyncMock()) - run = SimpleNamespace(status="running", winner_agent=None, rounds_completed=1) - backend = SimpleNamespace(update_run=AsyncMock()) - - try: - with caplog.at_level("WARNING"): - finalized = await optimizer._finalize( - workspace="default", - backend=backend, - agents_dir=tmp_path / "eval-and-optimize" / "agents", - run_entity=run, - evolution_tree=tree, - agent_name="agent", - insight_dataset=_insight_dataset([Task(id="task-a")]), - ) - finally: - type.__setattr__( - EvolutionaryOptimizer, - "write_final_report", - original_report_writer, - ) - - assert finalized is winner - assert run.status == "completed" - assert run.winner_agent == winner.label - backend.update_run.assert_awaited_once() - assert "Skipping Insight Suite report sections" in caplog.text diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_model_config.py b/plugins/nemo-experimentalist/tests/experimentalist/test_model_config.py index 90e643490e..9f20928b3f 100644 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_model_config.py +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_model_config.py @@ -221,12 +221,12 @@ def test_record_reward_merges_rather_than_replaces() -> None: from nemo_experimentalist_plugin.entities import Candidate candidate = Candidate(run_id="run-1", label="agent-0", round=0, optimization="baseline") - candidate.record_reward("insight", metrics={"reward": 0.5}) - candidate.record_reward("insight", metadata={"suite_identity": "sha256:abc"}) + candidate.record_reward("custom", metrics={"reward": 0.5}) + candidate.record_reward("custom", metadata={"source": "external"}) - record = candidate.reward("insight") + record = candidate.reward("custom") assert record.metrics == {"reward": 0.5} - assert record.metadata == {"suite_identity": "sha256:abc"} + assert record.metadata == {"source": "external"} def test_an_unmeasured_channel_is_distinguishable_from_an_empty_one() -> None: diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_reporting.py b/plugins/nemo-experimentalist/tests/experimentalist/test_reporting.py index 1c5b69fe77..678e634de5 100644 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_reporting.py +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_reporting.py @@ -8,11 +8,13 @@ import io from pathlib import Path -from nemo_experimentalist_plugin.experimentalist.reporting import ( - RunReporter, - Verbosity, - reward_scalar, -) +from nemo_experimentalist_plugin.config import MetricTarget +from nemo_experimentalist_plugin.experimentalist.reporting import RunReporter, Verbosity + +OBJECTIVES = [ + MetricTarget(name="success", direction="maximize"), + MetricTarget(name="tokens", direction="minimize"), +] def _reporter(verbosity: Verbosity = Verbosity.NORMAL) -> tuple[RunReporter, io.StringIO]: @@ -46,19 +48,28 @@ def test_progress_renders_fraction_and_bare_phase() -> None: assert "proposing candidates" in out -def test_candidate_evaluated_sets_baseline_then_shows_delta() -> None: +def test_candidate_evaluated_shows_all_objectives_and_direction_aware_deltas() -> None: r, sink = _reporter() r.candidate_evaluated( - label="agent-0", split="validation", reward=0.05, artifacts=Path("/exp/results/agent-0-validation") + label="agent-0", + split="validation", + metrics={"success": 0.05, "tokens": 100.0}, + objective_metrics=OBJECTIVES, + artifacts=Path("/exp/results/agent-0-validation"), ) r.candidate_evaluated( - label="agent-1", split="validation", reward=0.19, artifacts=Path("/exp/results/agent-1-validation") + label="agent-1", + split="validation", + metrics={"success": 0.19, "tokens": 80.0}, + objective_metrics=OBJECTIVES, + artifacts=Path("/exp/results/agent-1-validation"), ) lines = sink.getvalue().splitlines() - assert "reward 0.050" in lines[0] - assert "▲" not in lines[0] # baseline has no delta - assert "reward 0.190" in lines[1] - assert "+0.140" in lines[1] # delta vs baseline + assert "success 0.050" in lines[0] + assert "tokens 100.000" in lines[0] + assert "▲" not in lines[0] + assert "success 0.190 ▲+0.140" in lines[1] + assert "tokens 80.000 ▲-20.000" in lines[1] def test_seed_baseline_sets_delta_reference_silently_for_resume() -> None: @@ -66,43 +77,66 @@ def test_seed_baseline_sets_delta_reference_silently_for_resume() -> None: # reference from its cached reward without emitting a line, so the first # newly evaluated candidate is measured against agent-0, not itself. r, sink = _reporter() - r.seed_baseline(0.05) + r.seed_baseline({"success": 0.05, "tokens": 100.0}) assert sink.getvalue() == "" # silent: no line emitted r.candidate_evaluated( - label="agent-1", split="validation", reward=0.19, artifacts=Path("/exp/results/agent-1-validation") + label="agent-1", + split="validation", + metrics={"success": 0.19, "tokens": 80.0}, + objective_metrics=OBJECTIVES, + artifacts=Path("/exp/results/agent-1-validation"), ) out = sink.getvalue() - assert "reward 0.190" in out + assert "success 0.190" in out assert "+0.140" in out # delta measured against the seeded 0.05 baseline def test_seed_baseline_is_noop_once_baseline_set() -> None: r, sink = _reporter() r.candidate_evaluated( - label="agent-0", split="validation", reward=0.05, artifacts=Path("/exp/results/agent-0-validation") + label="agent-0", + split="validation", + metrics={"success": 0.05, "tokens": 100.0}, + objective_metrics=OBJECTIVES, + artifacts=Path("/exp/results/agent-0-validation"), ) - r.seed_baseline(0.99) # must not clobber the real baseline + r.seed_baseline({"success": 0.99, "tokens": 1.0}) # must not clobber the real baseline r.candidate_evaluated( - label="agent-1", split="validation", reward=0.19, artifacts=Path("/exp/results/agent-1-validation") + label="agent-1", + split="validation", + metrics={"success": 0.19, "tokens": 80.0}, + objective_metrics=OBJECTIVES, + artifacts=Path("/exp/results/agent-1-validation"), ) assert "+0.140" in sink.getvalue() # still measured against 0.05, not 0.99 def test_train_split_shows_no_delta() -> None: r, sink = _reporter() - r.candidate_evaluated(label="agent-0", split="train", reward=0.23, artifacts=Path("/exp/results/agent-0-train")) + r.candidate_evaluated( + label="agent-0", + split="train", + metrics={"success": 0.23, "tokens": 50.0}, + objective_metrics=OBJECTIVES, + artifacts=Path("/exp/results/agent-0-train"), + ) assert "▲" not in sink.getvalue() assert "▼" not in sink.getvalue() def test_run_finished_winner_and_no_winner() -> None: r, sink = _reporter() - r.candidate_evaluated(label="agent-0", split="validation", reward=0.05, artifacts=Path("/x")) - r.run_finished(winner="agent-1", scores={"reward": 0.19}, report_path=Path("/exp/final_report.md")) + r.candidate_evaluated( + label="agent-0", + split="validation", + metrics={"success": 0.05, "tokens": 100.0}, + objective_metrics=OBJECTIVES, + artifacts=Path("/x"), + ) + r.run_finished(winner="agent-1", scores={"success": 0.19, "tokens": 80.0}, report_path=Path("/exp/final_report.md")) out = sink.getvalue() assert "winner=agent-1" in out - assert "validation 0.190" in out - assert "baseline 0.050" in out + assert "validation success 0.190, tokens 80.000" in out assert "final_report.md" in out r2, sink2 = _reporter() @@ -114,13 +148,19 @@ def test_quiet_suppresses_phase_and_candidate_but_keeps_header_footer() -> None: r, sink = _reporter(Verbosity.QUIET) r.run_started(run_dir=Path("/x"), agent="a", insight=None, strategy="evolutionary") r.progress(phase="baseline", completed=0, total=15) - r.candidate_evaluated(label="agent-0", split="validation", reward=0.05, artifacts=Path("/x")) - r.run_finished(winner="agent-0", scores={"reward": 0.05}, report_path=None) + r.candidate_evaluated( + label="agent-0", + split="validation", + metrics={"success": 0.05, "tokens": 100.0}, + objective_metrics=OBJECTIVES, + artifacts=Path("/x"), + ) + r.run_finished(winner="agent-0", scores={"success": 0.05, "tokens": 100.0}, report_path=None) out = sink.getvalue() assert "strategy=evolutionary" in out # header kept assert "winner=agent-0" in out # footer kept assert "baseline" not in out # phase suppressed - assert "reward 0.050" not in out # candidate suppressed + assert "success 0.050" not in out # candidate suppressed def test_methods_never_raise_on_broken_sink() -> None: @@ -133,16 +173,17 @@ def write(self, s: str) -> int: # type: ignore[override] r.run_started(run_dir=Path("/x"), agent="a", insight=None, strategy="s") r.progress(phase="p", completed=1, total=2) r.candidate_started(label="agent-1", optimization="x", i=1, n=3) - r.candidate_evaluated(label="agent-1", split="validation", reward=0.1, artifacts=Path("/x")) - r.run_finished(winner="agent-1", scores={"reward": 0.1}, report_path=None) + r.candidate_evaluated( + label="agent-1", + split="validation", + metrics={"success": 0.1, "tokens": 10.0}, + objective_metrics=OBJECTIVES, + artifacts=Path("/x"), + ) + r.run_finished(winner="agent-1", scores={"success": 0.1, "tokens": 10.0}, report_path=None) r.note("hello") -def test_reward_scalar_defaults_to_zero_when_absent() -> None: - assert reward_scalar({"reward": 0.42}) == 0.42 - assert reward_scalar({}) == 0.0 - - def test_candidate_evaluated_renders_subset_style_result_id_verbatim() -> None: sink = io.StringIO() r = RunReporter(sink=sink) @@ -150,7 +191,8 @@ def test_candidate_evaluated_renders_subset_style_result_id_verbatim() -> None: r.candidate_evaluated( label="agent-1", split="train", - reward=0.2, + metrics={"success": 0.2, "tokens": 20.0}, + objective_metrics=OBJECTIVES, artifacts=Path("/exp/results/agent-1-train-subset-1-42e2eab5a4e0"), ) assert "agent-1-train-subset-1-42e2eab5a4e0" in sink.getvalue() @@ -163,15 +205,29 @@ def test_full_run_transcript_is_the_loop_emission_contract() -> None: r.run_started(run_dir=Path("/exp/run-1"), agent="nemo-oo-airline", insight="insight-892a", strategy="evolutionary") r.progress(phase="baseline", completed=0, total=15) r.candidate_evaluated( - label="agent-0", split="validation", reward=0.05, artifacts=Path("/exp/results/agent-0-validation") + label="agent-0", + split="validation", + metrics={"success": 0.05, "tokens": 100.0}, + objective_metrics=OBJECTIVES, + artifacts=Path("/exp/results/agent-0-validation"), + ) + r.candidate_evaluated( + label="agent-0", + split="train", + metrics={"success": 0.23, "tokens": 50.0}, + objective_metrics=OBJECTIVES, + artifacts=Path("/exp/results/agent-0-train"), ) - r.candidate_evaluated(label="agent-0", split="train", reward=0.23, artifacts=Path("/exp/results/agent-0-train")) r.progress(phase="evaluating candidates", completed=1, total=15) r.candidate_started(label="agent-1", optimization="ground tool signatures", i=1, n=3) r.candidate_evaluated( - label="agent-1", split="validation", reward=0.19, artifacts=Path("/exp/results/agent-1-validation") + label="agent-1", + split="validation", + metrics={"success": 0.19, "tokens": 80.0}, + objective_metrics=OBJECTIVES, + artifacts=Path("/exp/results/agent-1-validation"), ) - r.run_finished(winner="agent-1", scores={"reward": 0.19}, report_path=Path("/exp/final_report.md")) + r.run_finished(winner="agent-1", scores={"success": 0.19, "tokens": 80.0}, report_path=Path("/exp/final_report.md")) out = sink.getvalue() # ordering sanity: baseline before round-1 before finish assert out.index("baseline") < out.index("evaluating candidates") < out.index("Finished") diff --git a/plugins/nemo-experimentalist/tests/experimentalist/test_terminator.py b/plugins/nemo-experimentalist/tests/experimentalist/test_terminator.py index a78cf5406a..4335c37c1b 100644 --- a/plugins/nemo-experimentalist/tests/experimentalist/test_terminator.py +++ b/plugins/nemo-experimentalist/tests/experimentalist/test_terminator.py @@ -14,6 +14,7 @@ from dataclasses import dataclass, field from types import SimpleNamespace +from nemo_experimentalist_plugin.config import MetricTarget from nemo_experimentalist_plugin.experimentalist.components.loop import EvolutionaryOptimizerConfig from nemo_experimentalist_plugin.experimentalist.components.terminator import ( TerminationDecision, @@ -108,7 +109,11 @@ async def test_run_stops_on_convergence_when_budget_not_hit() -> None: round_num=2, evolution_tree=tree, prior_analysis="prior analysis", - config=EvolutionaryOptimizerConfig(max_rounds=15, min_rounds_before_stopping=2), + config=EvolutionaryOptimizerConfig( + max_rounds=15, + min_rounds_before_stopping=2, + objective_function=[MetricTarget(name="score", direction="maximize")], + ), ) assert decision.stop is True assert "converged" in decision.reason @@ -214,6 +219,7 @@ async def test_has_converged_true_when_front_stagnates() -> None: evolution_tree=tree, prior_analysis="ignored", min_rounds_before_stopping=2, + objective_metrics=[MetricTarget(name="score", direction="maximize")], ) is True ) @@ -238,6 +244,7 @@ async def test_qualitative_fallback_true() -> None: evolution_tree=tree, prior_analysis="plateaued: same root cause every round", min_rounds_before_stopping=2, + objective_metrics=[MetricTarget(name="score", direction="maximize")], ) is True ) diff --git a/plugins/nemo-experimentalist/tests/test_experiment_mirror.py b/plugins/nemo-experimentalist/tests/test_experiment_mirror.py index 4d9b9da8e9..559517159c 100644 --- a/plugins/nemo-experimentalist/tests/test_experiment_mirror.py +++ b/plugins/nemo-experimentalist/tests/test_experiment_mirror.py @@ -117,18 +117,18 @@ async def test_project_candidate_skips_when_no_reward(): experiments.create.assert_not_awaited() -async def test_project_candidate_creates_insight_experiment_when_evaluated(): +async def test_project_candidate_creates_experiment_for_custom_reward_channel(): experiments = AsyncMock() - experiments.create.return_value = SimpleNamespace(id="exp-insight") + experiments.create.return_value = SimpleNamespace(id="exp-custom") mirror = ExperimentMirror(_client(AsyncMock(), experiments), workspace="default") - candidate = _cand(rewards={"insight": RewardRecord(metrics={"uses_required_tool": 0.5}, trials=[])}) + candidate = _cand(rewards={"custom": RewardRecord(metrics={"custom_score": 0.5}, trials=[])}) await mirror.project_candidate(candidate) kwargs = experiments.create.await_args.kwargs - assert kwargs["name"] == "opt-run-1-agent-0-insight" - assert kwargs["dataset_name"] == "insight" - assert kwargs["metadata"] == {"round": "0", "candidate_id": "agent-0", "split": "insight"} + assert kwargs["name"] == "opt-run-1-agent-0-custom" + assert kwargs["dataset_name"] == "custom" + assert kwargs["metadata"] == {"round": "0", "candidate_id": "agent-0", "split": "custom"} async def test_project_candidate_conflict_updates_experiment(): diff --git a/plugins/nemo-experimentalist/tests/test_experimentalist_analyzer.py b/plugins/nemo-experimentalist/tests/test_experimentalist_analyzer.py index 48ab2c4f44..7bac381067 100644 --- a/plugins/nemo-experimentalist/tests/test_experimentalist_analyzer.py +++ b/plugins/nemo-experimentalist/tests/test_experimentalist_analyzer.py @@ -22,11 +22,13 @@ import pytest from nemo_experimentalist_plugin.experimentalist.components import analyzer as analyzer_module +from nemo_experimentalist_plugin.experimentalist.components import cache from nemo_experimentalist_plugin.experimentalist.components.analyzer import ( AgentAnalyzer, AnalyzerConfig, FailureClassification, PeerComparison, + TrialSelection, ) from nemo_experimentalist_plugin.experimentalist.components.rationalizer import Rationale from nemo_experimentalist_plugin.experimentalist.components.trace_analyzer import Diagnostic @@ -62,6 +64,7 @@ def list_tasks(self) -> list[_FakeTask]: class _FakeEvaluation: id: str = "eval-1" aggregate_metrics: dict[str, float] = field(default_factory=lambda: {"reward": 1.0}) + trials: list[_FakeTrial] = field(default_factory=list) class _RecordingTraceAnalyzer: @@ -85,10 +88,21 @@ async def run( agent_path: Any, rationale: Any = None, insight: Any = None, + selection_reason: str = "", + objective_metrics: list[dict[str, str]] | None = None, + regression_metrics: list[dict[str, str]] | None = None, client: Any = None, workspace: Any = None, ) -> Diagnostic: - type(self).calls.append({"client": client, "workspace": workspace}) + type(self).calls.append( + { + "client": client, + "workspace": workspace, + "selection_reason": selection_reason, + "objective_metrics": objective_metrics, + "regression_metrics": regression_metrics, + } + ) return Diagnostic(outcome="SUCCESS", summary="stub", failure_point=None, root_cause="stub") @@ -108,18 +122,40 @@ class _SelectTrials: def __init__(self, trials: list[Any]) -> None: self._trials = trials - async def __call__(self, agent_id: str, dataset: Any, evaluation: Any) -> list[Any]: - return self._trials + async def __call__( + self, + agent_id: str, + dataset: Any, + evaluation: Any, + objective_metrics: list[dict[str, str]], + regression_metrics: list[dict[str, str]], + ) -> list[TrialSelection]: + return [ + TrialSelection(trial_id=trial.id, reason=f"Analyze {trial.id} for this test.") for trial in self._trials + ] class _ClassifyFailures: - async def __call__(self, agent_id: str, diagnoses: Any, trials: Any) -> FailureClassification: + async def __call__( + self, + agent_id: str, + diagnoses: Any, + trials: Any, + objective_metrics: list[dict[str, str]], + regression_metrics: list[dict[str, str]], + ) -> FailureClassification: return FailureClassification(systematic=[], mechanical=[]) class _CompareWithPeers: async def __call__( - self, agent_id: str, evaluation: Any, diagnoses: Any, peer_evaluations: Any = None + self, + agent_id: str, + evaluation: Any, + diagnoses: Any, + peer_evaluations: Any = None, + objective_metrics: list[dict[str, str]] | None = None, + regression_metrics: list[dict[str, str]] | None = None, ) -> PeerComparison: return PeerComparison(divergent_trials=[], complementary_patterns=[]) @@ -148,7 +184,32 @@ class _TA(_RecordingTraceAnalyzer): def _fixtures() -> tuple[_FakeTrial, _FakeDataset, _FakeEvaluation]: trial = _FakeTrial(id="trial-1", task_id="task-1", trace=object(), metrics={"reward": _FakeMetric(0.0)}) dataset = _FakeDataset(tasks=[_FakeTask(id="task-1")]) - return trial, dataset, _FakeEvaluation() + return trial, dataset, _FakeEvaluation(trials=[trial]) + + +def test_trace_cache_key_uses_trace_uri_namespace() -> None: + """Trace-analysis cache files must not be named as agent-analysis cache files.""" + assert cache.trace_uri_hash("intake://trace-1:objective-metrics:[]").startswith("trace-uri-") + + +def test_peer_comparison_respects_minimize_metric_directions(tmp_path: Path) -> None: + analyzer = _make_analyzer(tmp_path, []) + focal = _FakeEvaluation( + trials=[_FakeTrial("focal", "task-1", None, {"quality": _FakeMetric(0.8), "tokens": _FakeMetric(10.0)})] + ) + peer = _FakeEvaluation( + trials=[_FakeTrial("peer", "task-1", None, {"quality": _FakeMetric(0.7), "tokens": _FakeMetric(5.0)})] + ) + directions = analyzer._metric_directions( + [{"name": "quality", "direction": "maximize"}, {"name": "tokens", "direction": "minimize"}], [] + ) + + pairs = analyzer._select_divergent_pairs("focal", focal, {"peer": peer}, directions) + complementary = analyzer._find_complementary_failures("focal", focal, {"peer": peer}, directions) + + assert pairs[0]["winner"] == "peer" + assert complementary["task-1"]["quality"]["leaders"] == ["focal"] + assert complementary["task-1"]["tokens"]["leaders"] == ["peer"] @pytest.mark.asyncio @@ -175,6 +236,9 @@ async def test_run_threads_client_and_workspace_into_trace_analyzer( assert len(calls) == 1 assert calls[0]["client"] is sentinel_client assert calls[0]["workspace"] == "tau2-airline-ws" + assert calls[0]["selection_reason"] == "Analyze trial-1 for this test." + assert calls[0]["objective_metrics"] == [] + assert calls[0]["regression_metrics"] == [] @pytest.mark.asyncio @@ -192,6 +256,29 @@ async def test_run_defaults_client_and_workspace_to_none(tmp_path: Path, monkeyp assert calls[0]["workspace"] is None +@pytest.mark.asyncio +async def test_run_threads_metric_contract_into_trace_analyzer(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Trace analysis receives the same objective and regression metrics as selection.""" + calls: list[dict[str, Any]] = [] + _install_fakes(monkeypatch, calls) + trial, dataset, evaluation = _fixtures() + analyzer = _make_analyzer(tmp_path, [trial]) + objective_metrics = [{"name": "success_rate", "direction": "maximize"}] + regression_metrics = [{"name": "tokens", "direction": "minimize"}] + + await analyzer.run( + agent="agent-a", + dataset=cast(Any, dataset), + evaluation=cast(Any, evaluation), + round=0, + objective_metrics=objective_metrics, + regression_metrics=regression_metrics, + ) + + assert calls[0]["objective_metrics"] == objective_metrics + assert calls[0]["regression_metrics"] == regression_metrics + + @pytest.mark.asyncio async def test_intake_availability_is_part_of_cache_key(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """A trace-skipped (no client) result must not be replayed once a client is available.""" diff --git a/plugins/nemo-experimentalist/tests/test_experimentalist_backend.py b/plugins/nemo-experimentalist/tests/test_experimentalist_backend.py index 1e07dcb857..241092bdb8 100644 --- a/plugins/nemo-experimentalist/tests/test_experimentalist_backend.py +++ b/plugins/nemo-experimentalist/tests/test_experimentalist_backend.py @@ -347,12 +347,12 @@ async def test_persist_result_preserves_generated_optimization_report(tmp_path: backend = _local_backend(tmp_path) report_path = backend._eo / "OPTIMIZATION.md" - report_path.write_text("# Full optimization report\n\nInsight Suite Metrics") + report_path.write_text("# Full optimization report\n\nAdditional Metrics") result = ExperimentalistResult(summary="compact run summary", run_id="run-1", rounds_completed=2, winner=None) await backend.persist_result(workspace="w", result=result) - assert report_path.read_text() == "# Full optimization report\n\nInsight Suite Metrics" + assert report_path.read_text() == "# Full optimization report\n\nAdditional Metrics" # --------------------------------------------------------------------------- diff --git a/plugins/nemo-experimentalist/tests/test_metric_contract.py b/plugins/nemo-experimentalist/tests/test_metric_contract.py new file mode 100644 index 0000000000..bb01578c81 --- /dev/null +++ b/plugins/nemo-experimentalist/tests/test_metric_contract.py @@ -0,0 +1,103 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest +from nemo_experimentalist_plugin.config import ( + EvolutionaryOptimizerConfig, + MetricTarget, + has_metric_dimensions, + pareto_objectives, +) +from nemo_experimentalist_plugin.experimentalist.components.loop import _with_insight_objective + + +def test_metric_contract_supports_multiple_objective_metrics() -> None: + config = EvolutionaryOptimizerConfig.model_validate( + { + "objective_function": [ + {"name": "tokens", "direction": "minimize"}, + {"name": "cost", "direction": "minimize"}, + ], + "regression_metrics": [{"name": "success_rate", "direction": "maximize"}], + } + ) + + assert config.optimization_policy() == ( + "Optimize these objective metric(s): tokens (minimize), cost (minimize). " + "Do not regress these metric(s): success_rate (maximize). " + "Metric values, including aggregates, are produced by the evaluator; do not invent formulas or weights." + ) + + +def test_metric_contract_treats_an_evaluator_aggregate_as_a_metric() -> None: + config = EvolutionaryOptimizerConfig.model_validate( + {"objective_function": [{"name": "quality", "direction": "maximize"}]} + ) + + assert [target.name for target in config.objective_function] == ["quality"] + + +def test_pareto_ranking_uses_only_objectives_and_normalizes_minimization() -> None: + config = EvolutionaryOptimizerConfig.model_validate( + { + "objective_function": [ + {"name": "tokens", "direction": "minimize"}, + {"name": "cost", "direction": "minimize"}, + ], + "regression_metrics": [{"name": "success", "direction": "maximize"}], + } + ) + + assert pareto_objectives({"tokens": 10.0, "cost": 2.0, "success": 0.9}, config.objective_function) == { + "tokens": -10.0, + "cost": -2.0, + } + + +def test_metric_dimensions_require_only_the_objectives_used_for_pareto_ranking() -> None: + objectives = [MetricTarget(name="quality", direction="maximize")] + regressions = [MetricTarget(name="cost", direction="minimize")] + + assert has_metric_dimensions({"quality": 0.9}, objectives) + assert not has_metric_dimensions({"cost": 8.0}, objectives) + assert has_metric_dimensions({"quality": 0.9, "cost": 11.0}, objectives) + assert not has_metric_dimensions({"quality": 0.9}, [*objectives, *regressions]) + + +@pytest.mark.parametrize( + "config", + [ + {"objective_function": []}, + { + "objective_function": [ + {"name": "reward", "direction": "maximize"}, + {"name": "reward", "direction": "minimize"}, + ] + }, + { + "objective_function": [{"name": "reward", "direction": "maximize"}], + "regression_metrics": [{"name": "reward", "direction": "maximize"}], + }, + ], +) +def test_metric_contract_rejects_ambiguous_or_overlapping_targets(config: dict[str, object]) -> None: + with pytest.raises(ValueError): + EvolutionaryOptimizerConfig.model_validate(config) + + +def test_insight_metrics_become_objectives_and_existing_targets_become_guardrails() -> None: + config = EvolutionaryOptimizerConfig.model_validate( + { + "objective_function": [{"name": "cost", "direction": "minimize"}], + "regression_metrics": [{"name": "safety", "direction": "maximize"}], + } + ) + + effective = _with_insight_objective(config, ("uses_required_tool", "cites_source")) + + assert [target.name for target in effective.objective_function] == ["uses_required_tool", "cites_source"] + assert all(target.direction == "maximize" for target in effective.objective_function) + assert [target.model_dump() for target in effective.regression_metrics] == [ + {"name": "cost", "direction": "minimize"}, + {"name": "safety", "direction": "maximize"}, + ]